Use native code for pthread_cleanup_push/pop. NFC

src/library.js

@@ -439,15 +439,6 @@ LibraryManager.library = {
   },
   __cxa_atexit: 'atexit',
 
-#endif
-
-  // used in rust, clang when doing thread_local statics
-#if USE_PTHREADS
-  __cxa_thread_atexit: 'pthread_cleanup_push',
-  __cxa_thread_atexit_impl: 'pthread_cleanup_push',
-#else
-  __cxa_thread_atexit: 'atexit',
-  __cxa_thread_atexit_impl: 'atexit',
 #endif
 
   // TODO: There are currently two abort() functions that get imported to asm module scope: the built-in runtime function abort(),

src/library_pthread.js

@@ -1058,15 +1058,12 @@ var LibraryPThread = {
     throw 'unwind';
   },
 
-  pthread_cleanup_push__sig: 'vii',
-  pthread_cleanup_push: function(routine, arg) {
-    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) });
+  __cxa_thread_atexit__sig: 'vii',
+  __cxa_thread_atexit: function(routine, arg) {
+    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) }); 
   },
+  __cxa_thread_atexit_impl: '__cxa_thread_atexit',
 
-  pthread_cleanup_pop: function(execute) {
-    var routine = PThread.threadExitHandlers.pop();
-    if (execute) routine();
-  },
 
   // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error.
   emscripten_futex_wait__deps: ['emscripten_main_thread_process_queued_calls'],

src/library_pthread_stub.js

@@ -17,23 +17,6 @@ var LibraryPThreadStub = {
 #endif
   },
 
-  pthread_cleanup_push__sig: 'vii',
-  pthread_cleanup_push: function(routine, arg) {
-    __ATEXIT__.push({ func: routine, arg: arg });
-    _pthread_cleanup_push.level = __ATEXIT__.length;
-  },
-
-  pthread_cleanup_pop__deps: ['pthread_cleanup_push'],
-  pthread_cleanup_pop__sig: 'vi',
-  pthread_cleanup_pop: function(execute) {
-    assert(_pthread_cleanup_push.level == __ATEXIT__.length, 'cannot pop if something else added meanwhile!');
-    var callback = __ATEXIT__.pop();
-    if (execute) {
-      {{{ makeDynCall('vi', 'callback.func') }}}(callback.arg)
-    }
-    _pthread_cleanup_push.level = __ATEXIT__.length;
-  },
-
   // When pthreads is not enabled, we can't use the Atomics futex api to do
   // proper sleeps, so simulate a busy spin wait loop instead.
   emscripten_thread_sleep__deps: ['emscripten_get_now'],
@@ -43,6 +26,9 @@ var LibraryPThreadStub = {
       // Do nothing.
     }
   },
+
+  __cxa_thread_atexit: 'atexit',
+  __cxa_thread_atexit_impl: 'atexit',
 };
 
 mergeInto(LibraryManager.library, LibraryPThreadStub);

system/lib/libc/musl/include/pthread.h

@@ -203,18 +203,11 @@ struct __ptcb {
 	struct __ptcb *__next;
 };
 
-#ifdef __EMSCRIPTEN__
-// For Emscripten, the cleanup stack is not implemented as a macro, since it's currently in the JS side.
-typedef void (*cleanup_handler_routine)(void *arg);
-void pthread_cleanup_push(cleanup_handler_routine routine, void *arg);
-void pthread_cleanup_pop(int execute);
-#else
 void _pthread_cleanup_push(struct __ptcb *, void (*)(void *), void *);
 void _pthread_cleanup_pop(struct __ptcb *, int);
 
 #define pthread_cleanup_push(f, x) do { struct __ptcb __cb; _pthread_cleanup_push(&__cb, f, x);
 #define pthread_cleanup_pop(r) _pthread_cleanup_pop(&__cb, (r)); } while(0)
-#endif
 
 #ifdef _GNU_SOURCE
 struct cpu_set_t;

system/lib/pthread/library_pthread.c

@@ -946,3 +946,34 @@ int emscripten_proxy_main(int argc, char** argv) {
 }
 
 weak_alias(__pthread_testcancel, pthread_testcancel);
+
+// See musl's pthread_create.c
+void __run_cleanup_handlers(void* _unused) {
+  pthread_t self = __pthread_self();
+  while (self->cancelbuf) {
+    void (*f)(void *) = self->cancelbuf->__f;
+    void *x = self->cancelbuf->__x;
+    self->cancelbuf = self->cancelbuf->__next;
+    f(x);
+  }
+}
+
+extern int __cxa_thread_atexit(void (*)(void *), void *, void *);
+
+extern int8_t __dso_handle;
+
+// Copied from musl's pthread_create.c
+void __do_cleanup_push(struct __ptcb *cb) {
+  struct pthread *self = __pthread_self();
+  cb->__next = self->cancelbuf;
+  self->cancelbuf = cb;
+  static thread_local bool registerd = false;
+  if (!registerd) {
+    __cxa_thread_atexit(__run_cleanup_handlers, NULL, &__dso_handle);
+  }
+}
+
+// Copied from musl's pthread_create.c
+void __do_cleanup_pop(struct __ptcb *cb) {
+  __pthread_self()->cancelbuf = cb->__next;
+}

tests/pthread/test_pthread_cleanup.cpp

@@ -92,6 +92,7 @@ int main() {
   printf("Cleanup state variable: %d", cleanupState);
   assert(cleanupState == 907640832);
 
+  pthread_cleanup_pop(1);
   exit(EXIT_SUCCESS);
 }
 

tests/pthread/test_pthread_cleanup.out

@@ -0,0 +1 @@
+Cleanup state variable: 907640832

tests/test_core.py

@@ -2336,6 +2336,12 @@ def test_pthread_thread_local_storage(self):
     self.set_setting('INITIAL_MEMORY', '300mb')
     self.do_run_in_out_file_test('pthread/test_pthread_thread_local_storage.cpp')
 
+  @node_pthreads
+  def test_pthread_cleanup(self):
+    self.set_setting('EXIT_RUNTIME')
+    self.set_setting('PTHREAD_POOL_SIZE', 4)
+    self.do_run_in_out_file_test('pthread/test_pthread_cleanup.cpp')
+
   def test_tcgetattr(self):
     self.do_runf(test_file('termios/test_tcgetattr.c'), 'success')
 

tools/system_libs.py

@@ -751,7 +751,7 @@ def get_files(self):
     if self.is_mt:
       ignore += [
         'clone.c', '__lock.c',
-        'pthread_cleanup_push.c', 'pthread_create.c',
+        'pthread_create.c',
         'pthread_kill.c', 'pthread_sigmask.c',
         '__set_thread_area.c', 'synccall.c',
         '__syscall_cp.c', '__tls_get_addr.c',
@@ -781,6 +781,7 @@ def get_files(self):
         path_components=['system', 'lib', 'libc', 'musl', 'src', 'thread'],
         filenames=[
           'pthread_self.c',
+          'pthread_cleanup_push.c',
           # C11 thread library functions
           'call_once.c',
           'tss_create.c',