From 0263daef86b973f40bb2a37d342b932fc7cba5f5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 09:27:22 -0700 Subject: [PATCH 01/25] wip [ci skip] need link to good length docs page --- src/settings.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/settings.js b/src/settings.js index 993f403c9d5fb..df17c54a148fe 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1298,6 +1298,9 @@ var PTHREAD_HINT_NUM_CORES = 4; // True when building with --threadprofiler var PTHREADS_PROFILING = 0; +// See ... +var ALLOW_PTHREAD_JOIN_ON_MAIN_THREAD = 0; + // If true, add in debug traces for diagnosing pthreads related issues. var PTHREADS_DEBUG = 0; From 9197b7911c64e7d2f85d4e22d25c6f96c4fe2fd9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 17:27:40 -0700 Subject: [PATCH 02/25] [ci skip] --- src/library_pthread.js | 1 + src/settings.js | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index b64313a623036..e8221bddd1201 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -1074,6 +1074,7 @@ var LibraryPThread = { // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], emscripten_futex_wait: function(addr, val, timeout) { +ALLOW_BLOCKING_ON_MAIN_THREAD if (addr <= 0 || addr > HEAP8.length || addr&3 != 0) return -{{{ cDefine('EINVAL') }}}; // dump('futex_wait addr:' + addr + ' by thread: ' + _pthread_self() + (ENVIRONMENT_IS_PTHREAD?'(pthread)':'') + '\n'); if (ENVIRONMENT_IS_WORKER) { diff --git a/src/settings.js b/src/settings.js index 10d0fc441924a..e2ae403f9179c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1294,8 +1294,9 @@ var PTHREAD_HINT_NUM_CORES = 4; // True when building with --threadprofiler var PTHREADS_PROFILING = 0; -// See ... -var ALLOW_PTHREAD_JOIN_ON_MAIN_THREAD = 0; +// By default we disallow running pthread_join This flag is ne.. +// https://emscripten.org/docs/porting/pthreads.html#special-considerations +var ALLOW_BLOCKING_ON_MAIN_THREAD = 0; // If true, add in debug traces for diagnosing pthreads related issues. var PTHREADS_DEBUG = 0; From 37d78fb12ecc9a12871fa3082a09a53ac7a96bdf Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 11:04:41 -0700 Subject: [PATCH 03/25] wip [ci skip] --- site/source/docs/porting/pthreads.rst | 40 ++++++++++++++++++++++++--- src/library_pthread.js | 4 ++- src/settings.js | 6 ++-- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index b999461ba764c..773aad4f33c69 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -43,6 +43,42 @@ but is unrelated. That flag does not use pthreads or SharedArrayBuffer, and instead uses a plain Web Worker to run your main program (and postMessage to proxy messages back and forth). +Blocking on the main browser thread +=================================== + +The Web API for atomics does not allow blocking on the main thread +(specifically, ``Atomics.wait`` doesn't work there). Such blocking is +necessary in APIs like ``pthread_join`` and anything that uses a futex wait +under the hood, like ``usleep()``, ``emscripten_futex_wait()``, or +``pthread_mutex_lock()``. Therefore by default all those don't work on the main +browser thread. + +Emscripten does have an option to allow blocking on the main browser thread:: + + -s ALLOW_BLOCKING_ON_MAIN_THREAD=1 + +When doing so we try to make those +blocking APIs work, but with serious downsides, so it is best to avoid +main thread blocking where possible (consider using ``PROXY_TO_PTHREAD`` as +mentioned earlier). The downsides include: + +- Some APIs are proxied to the main browser thread because of Web limitations. If + the main thread blocks while a worker attempts to proxy to it, a deadlock + can occur. In particular, DOM access is only possible on the main browser + thread, and therefore things like filesystem operations (``fopen()``, + etc.) or changing the HTML page's title, etc., are proxied over to the main + browser thread. See + `bug 3495 `_ for + more information and how to try to work around this until this. To check which + operations are proxied, you can look for the function's implementation in + the JS library (``src/library_*``) and see if it is annotated with + ``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser + itself proxies certain things (like some GL operations), so there is no + general way to be safe here (aside from not blocking on the main browser + thread). +- To simulate a blocking wait on the main thread, we are forced to busy-wait, + which is bad because it freezes the tab and also wastes power. + Special considerations ====================== @@ -50,10 +86,6 @@ The Emscripten implementation for the pthreads API should follow the POSIX stand - When the linker flag ``-s PTHREAD_POOL_SIZE=`` is not specified and ``pthread_create()`` is called, the new thread will not start until control is yielded back to the browser's main event loop, because the web worker cannot be created while JS or wasm code is running. This is a violation of POSIX behavior and will break common code which creates a thread and immediately joins it or otherwise synchronously waits to observe an effect such as a memory write. Using a pool creates the web workers before main is called, allowing thread creation to be synchronous. -- Browser DOM access is only possible on the main browser thread, and therefore things that may access the DOM, like filesystem operations (``fopen()``, etc.) or changing the HTML page's title, etc., are proxied over to the main browser thread. This proxying can generate a deadlock in a special situation that native code running pthreads does not have. See `bug 3495 `_ for more information and how to work around this until this proxying is no longer needed in Emscripten. (To check which operations are proxied, you can look for the function's implementation in the JS library (``src/library_*``) and see if it is annotated with ``__proxy: 'sync'`` or ``__proxy: 'async'``.) - -- When doing a futex wait, e.g. ``usleep()``, ``emscripten_futex_wait()``, or ``pthread_mutex_lock()``, we use ``Atomics.wait`` on workers, which the browser should do pretty efficiently. But that is not available on the main thread, and so we busy-wait there. Busy-waiting is not recommended because it freezes the tab, and also wastes power. - - The Emscripten implementation does not support `POSIX signals `_, which are sometimes used in conjunction with pthreads. This is because it is not possible to send signals to web workers and pre-empt their execution. The only exception to this is pthread_kill() which can be used as normal to forcibly terminate a running thread. - The Emscripten implementation does also not support multiprocessing via ``fork()`` and ``join()``. diff --git a/src/library_pthread.js b/src/library_pthread.js index e8221bddd1201..1a42188156169 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -1074,7 +1074,6 @@ var LibraryPThread = { // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], emscripten_futex_wait: function(addr, val, timeout) { -ALLOW_BLOCKING_ON_MAIN_THREAD if (addr <= 0 || addr > HEAP8.length || addr&3 != 0) return -{{{ cDefine('EINVAL') }}}; // dump('futex_wait addr:' + addr + ' by thread: ' + _pthread_self() + (ENVIRONMENT_IS_PTHREAD?'(pthread)':'') + '\n'); if (ENVIRONMENT_IS_WORKER) { @@ -1091,6 +1090,9 @@ ALLOW_BLOCKING_ON_MAIN_THREAD if (ret === 'ok') return 0; throw 'Atomics.wait returned an unexpected value ' + ret; } else { +#if !ALLOW_BLOCKING_ON_MAIN_THREAD + abort('Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); +#endif // Atomics.wait is not available in the main browser thread, so simulate it via busy spinning. var loadedVal = Atomics.load(HEAP32, addr >> 2); if (val != loadedVal) return -{{{ cDefine('EWOULDBLOCK') }}}; diff --git a/src/settings.js b/src/settings.js index e2ae403f9179c..025647739702e 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1294,8 +1294,10 @@ var PTHREAD_HINT_NUM_CORES = 4; // True when building with --threadprofiler var PTHREADS_PROFILING = 0; -// By default we disallow running pthread_join This flag is ne.. -// https://emscripten.org/docs/porting/pthreads.html#special-considerations +// By default we disallow running pthread_join and other blocking operations +// on the main thread, as doing so can cause deadlocks on the Web (and also +// it works using a busy-wait which is expensive). See +// https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread var ALLOW_BLOCKING_ON_MAIN_THREAD = 0; // If true, add in debug traces for diagnosing pthreads related issues. From 68e1a88e9862ea61d2e2fab6d50c2020c9d84d25 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 12:50:55 -0700 Subject: [PATCH 04/25] [ci skip] --- ChangeLog.md | 4 +++ tests/test_browser.py | 64 +++++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 7813661e565d1..ecc6c91c48c7c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,6 +17,10 @@ See docs/process.md for how version tagging works. Current Trunk ------------- + - Pthread blocking on the main thread is now disallowed by default. To enable + it, you must set `ALLOW_BLOCKING_ON_MAIN_THREAD` (disallowing by default + makes it less likely new users of pthreads will hit surprising deadlocks + on the Web). v.1.38.47: 10/02/2019 --------------------- diff --git a/tests/test_browser.py b/tests/test_browser.py index 0b8f132b65df1..20c04cb780e83 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1913,7 +1913,7 @@ def test_gl_glteximage(self): @requires_graphics_hardware @requires_threads def test_gl_textures(self): - for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1']]: + for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']]: self.btest('gl_textures.cpp', '0', args=['-lGL'] + args) @requires_graphics_hardware @@ -3595,12 +3595,12 @@ def prep_no_SAB(self): # Test that the emscripten_ atomics api functions work. @requires_threads def test_pthread_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test 64-bit atomics. @requires_threads def test_pthread_64bit_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test 64-bit C++11 atomics. @requires_threads @@ -3621,40 +3621,40 @@ def test_pthread_gcc_atomic_fetch_and_op(self): print(args) if self.is_wasm_backend() and '--separate-asm' in args or 'AGGRESSIVE_VARIABLE_ELIMINATION=1' in args: continue - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_fetch_and_op.cpp'), expected='0', args=args + ['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_fetch_and_op.cpp'), expected='0', args=args + ['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # 64 bit version of the above test. @requires_threads def test_pthread_gcc_64bit_atomic_fetch_and_op(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_fetch_and_op.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_fetch_and_op.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) # Test the old GCC atomic __sync_op_and_fetch builtin operations. @requires_threads def test_pthread_gcc_atomic_op_and_fetch(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) # 64 bit version of the above test. @requires_threads def test_pthread_gcc_64bit_atomic_op_and_fetch(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) # Tests the rest of the remaining GCC atomics after the two above tests. @requires_threads def test_pthread_gcc_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test the __sync_lock_test_and_set and __sync_lock_release primitives. @requires_threads def test_pthread_gcc_spinlock(self): for arg in [[], ['-DUSE_EMSCRIPTEN_INTRINSICS']]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_spinlock.cpp'), expected='800', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + arg, also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_spinlock.cpp'), expected='800', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + arg, also_asmjs=True) # Test that basic thread creation works. @requires_threads def test_pthread_create(self): def test(args): print(args) - self.btest(path_from_root('tests', 'pthread', 'test_pthread_create.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + args) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_create.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + args) test([]) test(['-O3']) @@ -3679,17 +3679,17 @@ def test_pthread_proxy_to_pthread(self): @requires_threads def test_pthread_create_pthread(self): for modularize in [[], ['-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', path_from_root('tests', 'shell_that_launches_modularize.html')]]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_create_pthread.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2'] + modularize) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_create_pthread.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + modularize) # Test another case of pthreads spawning pthreads, but this time the callers immediately join on the threads they created. @requires_threads def test_pthread_nested_spawns(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that main thread can wait for a pthread to finish via pthread_join(). @requires_threads def test_pthread_join(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_join.cpp'), expected='6765', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_join.cpp'), expected='6765', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that threads can rejoin the pool once detached and finished @requires_threads @@ -3710,37 +3710,37 @@ def test_pthread_kill(self): # Test that pthread cleanup stack (pthread_cleanup_push/_pop) works. @requires_threads def test_pthread_cleanup(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_cleanup.cpp'), expected='907640832', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_cleanup.cpp'), expected='907640832', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Tests the pthread mutex api. @requires_threads def test_pthread_mutex(self): for arg in [[], ['-DSPINLOCK_TEST']]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + arg) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + arg) @requires_threads def test_pthread_attr_getstack(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_attr_getstack.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_attr_getstack.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that memory allocation is thread-safe. @requires_threads def test_pthread_malloc(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Stress test pthreads allocating memory that will call to sbrk(), and main thread has to free up the data. @requires_threads def test_pthread_malloc_free(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=256MB']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=256MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that the pthread_barrier API works ok. @requires_threads def test_pthread_barrier(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_barrier.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_barrier.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test the pthread_once() function. @requires_threads def test_pthread_once(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_once.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_once.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test against a certain thread exit time handling bug by spawning tons of threads. @requires_threads @@ -3762,13 +3762,13 @@ def test_pthread_thread_local_storage(self): # Test the pthread condition variable creation and waiting. @requires_threads def test_pthread_condition_variable(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_condition_variable.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_condition_variable.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that pthreads are able to do printf. @requires_threads def test_pthread_printf(self): def run(debug): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_printf.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'LIBRARY_DEBUG=%d' % debug]) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_printf.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'LIBRARY_DEBUG=%d' % debug, '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) run(debug=True) run(debug=False) @@ -3776,7 +3776,7 @@ def run(debug): # Test that pthreads are able to do cout. Failed due to https://bugzilla.mozilla.org/show_bug.cgi?id=1154858. @requires_threads def test_pthread_iostream(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_iostream.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_iostream.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that the main thread is able to use pthread_set/getspecific. @requires_threads @@ -3791,12 +3791,12 @@ def test_pthread_num_logical_cores(self): # Test that pthreads have access to filesystem. @requires_threads def test_pthread_file_io(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_file_io.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_file_io.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that the pthread_create() function operates benignly in the case that threading is not supported. @requires_threads def test_pthread_supported(self): - for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']]: + for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']]: self.btest(path_from_root('tests', 'pthread', 'test_pthread_supported.cpp'), expected='0', args=['-O3'] + args) # Test that --separate-asm works with -s USE_PTHREADS=1. @@ -3853,7 +3853,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that if the main thread is performing a futex wait while a pthread needs it to do a proxied operation (before that pthread would wake up the main thread), that it's not a deadlock. @requires_threads def test_pthread_proxying_in_futex_wait(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_proxying_in_futex_wait.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_proxying_in_futex_wait.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that sbrk() operates properly in multithreaded conditions @requires_threads @@ -3862,7 +3862,7 @@ def test_pthread_sbrk(self): print('aborting malloc=' + str(aborting_malloc)) # With aborting malloc = 1, test allocating memory in threads # With aborting malloc = 0, allocate so much memory in threads that some of the allocations fail. - self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=128MB']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=128MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that -s ABORTING_MALLOC=0 works in both pthreads and non-pthreads builds. (sbrk fails gracefully) @requires_threads @@ -3874,12 +3874,12 @@ def test_pthread_gauge_available_memory(self): # Test that the proxying operations of user code from pthreads to main thread work @requires_threads def test_pthread_run_on_main_thread(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test how a lot of back-to-back called proxying operations behave. @requires_threads def test_pthread_run_on_main_thread_flood(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread_flood.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread_flood.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that it is possible to synchronously call a JavaScript function on the main thread and get a return value back. @requires_threads @@ -3916,7 +3916,7 @@ def test_pthread_clock_drift(self): @requires_threads def test_pthread_utf8_funcs(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_utf8_funcs.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_utf8_funcs.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test the emscripten_futex_wake(addr, INT_MAX); functionality to wake all waiters @requires_threads @@ -4462,7 +4462,7 @@ def test_asmfs_relative_paths(self): def test_pthread_locale(self): for args in [ [], - ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2'], + ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], ]: print("Testing with: ", args) self.btest('pthread/test_pthread_locale.c', expected='1', args=args) @@ -4642,7 +4642,7 @@ def test_unicode_html_shell(self): # Tests the functionality of the emscripten_thread_sleep() function. @requires_threads def test_emscripten_thread_sleep(self): - self.btest(path_from_root('tests', 'pthread', 'emscripten_thread_sleep.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["print"]']) + self.btest(path_from_root('tests', 'pthread', 'emscripten_thread_sleep.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["print"]', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Tests that Emscripten-compiled applications can be run from a relative path in browser that is different than the address of the current page def test_browser_run_from_different_directory(self): From 27b72cca7a32f1feffb29321887ec5012df8d9c0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 13:11:05 -0700 Subject: [PATCH 05/25] test --- tests/pthread/main_thread_blocking.cpp | 64 ++++++++++++++++++++++++++ tests/test_browser.py | 7 +++ 2 files changed, 71 insertions(+) create mode 100644 tests/pthread/main_thread_blocking.cpp diff --git a/tests/pthread/main_thread_blocking.cpp b/tests/pthread/main_thread_blocking.cpp new file mode 100644 index 0000000000000..943818849c969 --- /dev/null +++ b/tests/pthread/main_thread_blocking.cpp @@ -0,0 +1,64 @@ +// Copyright 2019 The Emscripten Authors. All rights reserved. +// Emscripten is available under two separate licenses, the MIT license and the +// University of Illinois/NCSA Open Source License. Both these licenses can be +// found in the LICENSE file. + +#include +#include +#include +#include + + +void *ThreadMain(void *arg) +{ + pthread_exit((void*)0); +} + +pthread_t CreateThread() +{ + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + pthread_t ret; + int rc = pthread_create(&ret, &attr, ThreadMain, (void*)0); + assert(rc == 0); + pthread_attr_destroy(&attr); + return ret; +} + +int main() { + if (!emscripten_has_threading_support()) + { +#ifdef REPORT_RESULT + REPORT_RESULT(0); +#endif + printf("Skipped: Threading is not supported.\n"); + return 0; + } + + int x = EM_ASM_INT({ + onerror = function(e) { + var message = e.toString(); + var success = message.indexOf("Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread") >= 0; + if (success && !Module.reported) { + Module.reported = true; + console.log("reporting success"); + // manually REPORT_RESULT; we shouldn't call back into native code at this point + var xhr = new XMLHttpRequest(); + xhr.open("GET", "http://localhost:8888/report_result?0"); + xhr.send(); + } + }; + return 0; + }); + + int status; + // This should fail on the main thread. + puts("trying to block..."); + pthread_join(CreateThread(), (void**)&status); + puts("blocked ok."); +#ifdef REPORT_RESULT + REPORT_RESULT(1); +#endif +} + diff --git a/tests/test_browser.py b/tests/test_browser.py index 20c04cb780e83..42e7bd5cc6254 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3609,6 +3609,13 @@ def test_pthread_64bit_cxx11_atomics(self): for pthreads in [[], ['-s', 'USE_PTHREADS=1']]: self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_cxx11_atomics.cpp'), expected='0', args=opt + pthreads + ['-std=c++11']) + @requires_threads + def test_pthread_main_thread_blocking(self): + # Test that without ALLOW_BLOCKING_ON_MAIN_THREAD we error on blocking on the main thread. + self.btest(path_from_root('tests', 'pthread', 'main_thread_blocking.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + # Of course everything works ok when we are on a pthread. + self.btest(path_from_root('tests', 'pthread', 'main_thread_blocking.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD']) + # Test the old GCC atomic __sync_fetch_and_op builtin operations. @requires_threads def test_pthread_gcc_atomic_fetch_and_op(self): From 2e5e63ed4a5c52e2cda8be546d068bf032f6e377 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 14:27:14 -0700 Subject: [PATCH 06/25] more tests --- tests/test_browser.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 42e7bd5cc6254..9994141f718d1 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3675,7 +3675,7 @@ def test_pthread_preallocates_workers(self): # Test that allocating a lot of threads doesn't regress. This needs to be checked manually! @requires_threads def test_pthread_large_pthread_allocation(self): - self.btest(path_from_root('tests', 'pthread', 'test_large_pthread_allocation.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=128MB', '-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=50'], message='Check output from test to ensure that a regression in time it takes to allocate the threads has not occurred.') + self.btest(path_from_root('tests', 'pthread', 'test_large_pthread_allocation.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=128MB', '-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=50', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], message='Check output from test to ensure that a regression in time it takes to allocate the threads has not occurred.') # Tests the -s PROXY_TO_PTHREAD=1 option. @requires_threads @@ -3691,7 +3691,7 @@ def test_pthread_create_pthread(self): # Test another case of pthreads spawning pthreads, but this time the callers immediately join on the threads they created. @requires_threads def test_pthread_nested_spawns(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test that main thread can wait for a pthread to finish via pthread_join(). @requires_threads @@ -3752,7 +3752,7 @@ def test_pthread_once(self): # Test against a certain thread exit time handling bug by spawning tons of threads. @requires_threads def test_pthread_spawns(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_spawns.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_spawns.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # It is common for code to flip volatile global vars for thread control. This is a bit lax, but nevertheless, test whether that # kind of scheme will work with Emscripten as well. @@ -4496,10 +4496,10 @@ def test_pthread_run_script(self): def test_emscripten_animate_canvas_element_size(self): for args in [ ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1'], - ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], - ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_EXPLICIT_CONTEXT_SWAP=1'], - ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], - ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_MANUALLY_SET_ELEMENT_CSS_SIZE=1'], + ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], + ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_EXPLICIT_CONTEXT_SWAP=1'], + ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], + ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_MANUALLY_SET_ELEMENT_CSS_SIZE=1'], ]: cmd = ['-lGL', '-O3', '-g2', '--shell-file', path_from_root('tests', 'canvas_animate_resize_shell.html'), '--separate-asm', '-s', 'GL_DEBUG=1', '--threadprofiler'] + args print(' '.join(cmd)) @@ -4517,7 +4517,7 @@ def test_pthread_hello_thread(self): @requires_threads def test_pthread_growth_mainthread(self): def run(emcc_args=[]): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth_mainthread.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB'] + emcc_args, also_asmjs=False) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth_mainthread.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + emcc_args, also_asmjs=False) run() run(['-s', 'MODULARIZE_INSTANCE=1']) @@ -4528,7 +4528,7 @@ def run(emcc_args=[]): @requires_threads def test_pthread_growth(self): def run(emcc_args=[]): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-g'] + emcc_args, also_asmjs=False) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-g', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + emcc_args, also_asmjs=False) run() run(['-s', 'ASSERTIONS=1']) From 905e668b8b98707f5535951b42fc550d6f097b11 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 14:29:14 -0700 Subject: [PATCH 07/25] more --- tests/test_browser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 9994141f718d1..112f934298fba 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3811,7 +3811,7 @@ def test_pthread_supported(self): @requires_threads def test_pthread_separate_asm_pthreads(self): for modularize in [[], ['-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', path_from_root('tests', 'shell_that_launches_modularize.html')]]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '--profiling'] + modularize) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '--separate-asm', '--profiling'] + modularize) # Test the operation of Module.pthreadMainPrefixURL variable @no_wasm_backend('uses js') @@ -3845,7 +3845,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile" string to locate where worker.js will be loaded from. create_test_file('shell.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-o', 'test.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1',, '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD' '-o', 'test.html']) shutil.move('test.worker.js', os.path.join('cdn', 'test.worker.js')) if self.is_wasm_backend(): shutil.copyfile('test.html.mem', os.path.join('cdn', 'test.html.mem')) @@ -3853,7 +3853,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile(foo)" function to locate where worker.js will be loaded from. create_test_file('shell2.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function(filename) { if (filename == "test.worker.js") return "cdn/test.worker.js"; else return filename; }, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell2.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-o', 'test2.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell2.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-o', 'test2.html']) try_delete('test.worker.js') self.run_browser('test2.html', '', '/report_result?1') From b24df05115341aec3f89f9f19098d74a353f8fd8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 14:34:48 -0700 Subject: [PATCH 08/25] fix --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 112f934298fba..a4ca19d4ec70a 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3845,7 +3845,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile" string to locate where worker.js will be loaded from. create_test_file('shell.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1',, '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD' '-o', 'test.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD' '-o', 'test.html']) shutil.move('test.worker.js', os.path.join('cdn', 'test.worker.js')) if self.is_wasm_backend(): shutil.copyfile('test.html.mem', os.path.join('cdn', 'test.html.mem')) From 37f275ed3f8be2b8aeb2e7e8f933e047571b80c0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 14:36:40 -0700 Subject: [PATCH 09/25] wip [ci skip] --- src/settings.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/settings.js b/src/settings.js index 025647739702e..a9835a00f4969 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1299,6 +1299,8 @@ var PTHREADS_PROFILING = 0; // it works using a busy-wait which is expensive). See // https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread var ALLOW_BLOCKING_ON_MAIN_THREAD = 0; +// XXX looks like the runtime has various futex_waits, some very short. +// those are safe. should we only complain about pthread_join? // If true, add in debug traces for diagnosing pthreads related issues. var PTHREADS_DEBUG = 0; From 12c191573980eb79ad3541ec6a31dfc6ed689008 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 16:15:16 -0700 Subject: [PATCH 10/25] fix --- src/library_pthread.js | 19 ++++++-- system/include/emscripten/threading.h | 4 ++ .../musl/src/thread/pthread_cond_timedwait.c | 6 +++ ...read_blocking.cpp => main_thread_join.cpp} | 0 tests/pthread/main_thread_wait.cpp | 47 +++++++++++++++++++ tests/test_browser.py | 16 +++++-- 6 files changed, 83 insertions(+), 9 deletions(-) rename tests/pthread/{main_thread_blocking.cpp => main_thread_join.cpp} (100%) create mode 100644 tests/pthread/main_thread_wait.cpp diff --git a/src/library_pthread.js b/src/library_pthread.js index 1a42188156169..eb900e38cde48 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -790,7 +790,16 @@ var LibraryPThread = { if (canceled == 2) throw 'Canceled!'; }, - {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait'], + emscripten_block_on_main_thread: function() { +#if ASSERTIONS + assert(ENVIRONMENT_IS_WEB); +#endif +#if !ALLOW_BLOCKING_ON_MAIN_THREAD + abort('Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); +#endif + }, + + {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_block_on_main_thread'], {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join: function(thread, status) { if (!thread) { err('pthread_join attempted on a null thread pointer!'); @@ -815,6 +824,11 @@ var LibraryPThread = { err('Attempted to join thread ' + thread + ', which was already detached!'); return ERRNO_CODES.EINVAL; // The thread is already detached, can no longer join it! } + + if (ENVIRONMENT_IS_WEB) { + _emscripten_block_on_main_thread(); + } + for (;;) { var threadStatus = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.threadStatus }}} ) >> 2); if (threadStatus == 1) { // Exited? @@ -1090,9 +1104,6 @@ var LibraryPThread = { if (ret === 'ok') return 0; throw 'Atomics.wait returned an unexpected value ' + ret; } else { -#if !ALLOW_BLOCKING_ON_MAIN_THREAD - abort('Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); -#endif // Atomics.wait is not available in the main browser thread, so simulate it via busy spinning. var loadedVal = Atomics.load(HEAP32, addr >> 2); if (val != loadedVal) return -{{{ cDefine('EWOULDBLOCK') }}}; diff --git a/system/include/emscripten/threading.h b/system/include/emscripten/threading.h index 339741634387b..3d66e5f739545 100644 --- a/system/include/emscripten/threading.h +++ b/system/include/emscripten/threading.h @@ -372,6 +372,10 @@ struct thread_profiler_block char name[32]; }; +// Called when blocking on the main thread. This will error if main thread +// blocking is not enabled, see ALLOW_BLOCKING_ON_MAIN_THREAD. +void emscripten_block_on_main_thread(void); + #ifdef __cplusplus } #endif diff --git a/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c b/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c index df35e0b1d07d2..407b2a8f73dfc 100644 --- a/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c +++ b/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c @@ -80,6 +80,12 @@ int __pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restri int e, seq, clock = c->_c_clock, cs, shared=0, oldstate, tmp; volatile int *fut; +#ifdef __EMSCRIPTEN__ + if (pthread_self() == emscripten_main_browser_thread_id()) { + emscripten_block_on_main_thread(); + } +#endif + if ((m->_m_type&15) && (m->_m_lock&INT_MAX) != __pthread_self()->tid) return EPERM; diff --git a/tests/pthread/main_thread_blocking.cpp b/tests/pthread/main_thread_join.cpp similarity index 100% rename from tests/pthread/main_thread_blocking.cpp rename to tests/pthread/main_thread_join.cpp diff --git a/tests/pthread/main_thread_wait.cpp b/tests/pthread/main_thread_wait.cpp new file mode 100644 index 0000000000000..132b63cfa37e1 --- /dev/null +++ b/tests/pthread/main_thread_wait.cpp @@ -0,0 +1,47 @@ +// Copyright 2019 The Emscripten Authors. All rights reserved. +// Emscripten is available under two separate licenses, the MIT license and the +// University of Illinois/NCSA Open Source License. Both these licenses can be +// found in the LICENSE file. + +#include +#include +#include +#include + +int main() { + if (!emscripten_has_threading_support()) + { +#ifdef REPORT_RESULT + REPORT_RESULT(0); +#endif + printf("Skipped: Threading is not supported.\n"); + return 0; + } + + int x = EM_ASM_INT({ + onerror = function(e) { + var message = e.toString(); + var success = message.indexOf("Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread") >= 0; + if (success && !Module.reported) { + Module.reported = true; + console.log("reporting success"); + // manually REPORT_RESULT; we shouldn't call back into native code at this point + var xhr = new XMLHttpRequest(); + xhr.open("GET", "http://localhost:8888/report_result?0"); + xhr.send(); + } + }; + return 0; + }); + + // This should fail on the main thread. + puts("trying to block..."); + pthread_cond_t cv = PTHREAD_COND_INITIALIZER; + pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + pthread_cond_wait(&cv, &lock); + puts("blocked ok."); +#ifdef REPORT_RESULT + REPORT_RESULT(1); +#endif +} + diff --git a/tests/test_browser.py b/tests/test_browser.py index a4ca19d4ec70a..d38af200d7815 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3609,12 +3609,18 @@ def test_pthread_64bit_cxx11_atomics(self): for pthreads in [[], ['-s', 'USE_PTHREADS=1']]: self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_cxx11_atomics.cpp'), expected='0', args=opt + pthreads + ['-std=c++11']) + @parameterized({ + 'join': ['join', True, True], + 'wait': ['wait', True, False], + }) @requires_threads - def test_pthread_main_thread_blocking(self): - # Test that without ALLOW_BLOCKING_ON_MAIN_THREAD we error on blocking on the main thread. - self.btest(path_from_root('tests', 'pthread', 'main_thread_blocking.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) - # Of course everything works ok when we are on a pthread. - self.btest(path_from_root('tests', 'pthread', 'main_thread_blocking.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD']) + def test_pthread_main_thread_blocking(self, name, check_main, check_worker): + if check_main: + print('Test that without ALLOW_BLOCKING_ON_MAIN_THREAD we error on blocking on the main thread.') + self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + if check_worker: + print('Test that everything works ok when we are on a pthread.') + self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD']) # Test the old GCC atomic __sync_fetch_and_op builtin operations. @requires_threads From 479058c47fdcfdc12f2de909285263158537ae1e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 16:29:25 -0700 Subject: [PATCH 11/25] text --- site/source/docs/porting/pthreads.rst | 68 +++++++++++++++++---------- src/settings.js | 4 +- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index 773aad4f33c69..add7e2720425b 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -50,34 +50,52 @@ The Web API for atomics does not allow blocking on the main thread (specifically, ``Atomics.wait`` doesn't work there). Such blocking is necessary in APIs like ``pthread_join`` and anything that uses a futex wait under the hood, like ``usleep()``, ``emscripten_futex_wait()``, or -``pthread_mutex_lock()``. Therefore by default all those don't work on the main -browser thread. - -Emscripten does have an option to allow blocking on the main browser thread:: +``pthread_mutex_lock()``. To make them work, we use a busy-wait on the main +browser thread, which can make the browser tab unresponsive, and also wastes +power. (On a pthread, this isn't a problem as it runs in a Web Worker, where +we don't need to busy-wait.) + +Busy-waiting on the main browser thread in general will work despite the +downsides just mentioned, for things like waiting on a mutex. +However, things like ``pthread_join`` and ``pthread_cond_wait`` +can block for long periods of time, and if that +happens on the main browser thread, and while other threads expect it to +respond, it can cause a surprising deadlock. That can happen because some APIs +are proxied to the main browser thread because of Web limitations, and if the +main thread blocks while a worker attempts to proxy to it, a deadlock can +occur. + +In particular, DOM access is only possible on the main browser +thread, and therefore things like filesystem operations (``fopen()``, +etc.) or changing the HTML page's title, etc., are proxied over to the main +browser thread. See +`bug 3495 `_ for +more information and how to try to work around this until this. To check which +operations are proxied, you can look for the function's implementation in +the JS library (``src/library_*``) and see if it is annotated with +``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser +itself proxies certain things (like some GL operations), so there is no +general way to be safe here (aside from not blocking on the main browser +thread). + +The bottom line is that on the Web it is bad for the main browser thread to +wait on anything else. Therefore by default Emscripten disallows +``pthread_join`` and ``pthread_cond_wait`` on the main browser thread, and +will throw an error (whose message will point to here). + +To avoid this problem, you can use ``PROXY_TO_PTHREAD``, which as +mentioned earlier moves your ``main()`` function to a pthread, which leaves +the main browser thread to focus only on receiving proxied events. This is +recommended in general, but may take some porting work, if the application +assumed ``main()`` was on the main browser thread. + +Alternatively, you can flip a flag to allow blocking on the main browser thread, +using:: -s ALLOW_BLOCKING_ON_MAIN_THREAD=1 -When doing so we try to make those -blocking APIs work, but with serious downsides, so it is best to avoid -main thread blocking where possible (consider using ``PROXY_TO_PTHREAD`` as -mentioned earlier). The downsides include: - -- Some APIs are proxied to the main browser thread because of Web limitations. If - the main thread blocks while a worker attempts to proxy to it, a deadlock - can occur. In particular, DOM access is only possible on the main browser - thread, and therefore things like filesystem operations (``fopen()``, - etc.) or changing the HTML page's title, etc., are proxied over to the main - browser thread. See - `bug 3495 `_ for - more information and how to try to work around this until this. To check which - operations are proxied, you can look for the function's implementation in - the JS library (``src/library_*``) and see if it is annotated with - ``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser - itself proxies certain things (like some GL operations), so there is no - general way to be safe here (aside from not blocking on the main browser - thread). -- To simulate a blocking wait on the main thread, we are forced to busy-wait, - which is bad because it freezes the tab and also wastes power. +This lets you avoid the error, but you have the risk of deadlocks and other +problems as mentioned earlier. Special considerations ====================== diff --git a/src/settings.js b/src/settings.js index c8389fecc0d34..4f7346df7170e 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1294,13 +1294,11 @@ var PTHREAD_HINT_NUM_CORES = 4; // True when building with --threadprofiler var PTHREADS_PROFILING = 0; -// By default we disallow running pthread_join and other blocking operations +// By default we disallow running pthread_join and pthread_cond_wait // on the main thread, as doing so can cause deadlocks on the Web (and also // it works using a busy-wait which is expensive). See // https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread var ALLOW_BLOCKING_ON_MAIN_THREAD = 0; -// XXX looks like the runtime has various futex_waits, some very short. -// those are safe. should we only complain about pthread_join? // If true, add in debug traces for diagnosing pthreads related issues. var PTHREADS_DEBUG = 0; From b366dc82f039acc147c9f4cde264331966a098ca Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 16:36:39 -0700 Subject: [PATCH 12/25] fix --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index d38af200d7815..f897210bf8920 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3770,7 +3770,7 @@ def test_pthread_volatile(self): # Test thread-specific data (TLS). @requires_threads def test_pthread_thread_local_storage(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_thread_local_storage.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ASSERTIONS=1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_thread_local_storage.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ASSERTIONS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) # Test the pthread condition variable creation and waiting. @requires_threads From 5ae5bbe7ee45039eb0a6bd2852ebcd5c73887f26 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 4 Oct 2019 17:27:25 -0700 Subject: [PATCH 13/25] fix typo --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index f897210bf8920..c55864b1a3df0 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3851,7 +3851,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile" string to locate where worker.js will be loaded from. create_test_file('shell.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD' '-o', 'test.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-o', 'test.html']) shutil.move('test.worker.js', os.path.join('cdn', 'test.worker.js')) if self.is_wasm_backend(): shutil.copyfile('test.html.mem', os.path.join('cdn', 'test.html.mem')) From b21d2c743443236d4287a22566d63e956f8be4cc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 14 Oct 2019 13:39:17 -0700 Subject: [PATCH 14/25] feedback --- tests/pthread/main_thread_join.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/pthread/main_thread_join.cpp b/tests/pthread/main_thread_join.cpp index 943818849c969..7d2e8e5b07bf1 100644 --- a/tests/pthread/main_thread_join.cpp +++ b/tests/pthread/main_thread_join.cpp @@ -16,13 +16,9 @@ void *ThreadMain(void *arg) pthread_t CreateThread() { - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); pthread_t ret; - int rc = pthread_create(&ret, &attr, ThreadMain, (void*)0); + int rc = pthread_create(&ret, NULL, ThreadMain, (void*)0); assert(rc == 0); - pthread_attr_destroy(&attr); return ret; } From 9d8c79b749aca8bfab2eadfb1a5494eb441c1012 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 14 Oct 2019 13:50:01 -0700 Subject: [PATCH 15/25] docs --- site/source/docs/porting/pthreads.rst | 55 ++++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index add7e2720425b..20729b47ec88c 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -43,9 +43,40 @@ but is unrelated. That flag does not use pthreads or SharedArrayBuffer, and instead uses a plain Web Worker to run your main program (and postMessage to proxy messages back and forth). +Proxying +======== + +The Web allows certain operations to only happen from the main browser thread, +like interacting with the DOM. As a result, various operations are proxied to +the main browser thread if they are called on a background thread. See +`bug 3495 `_ for +more information and how to try to work around this until this. To check which +operations are proxied, you can look for the function's implementation in +the JS library (``src/library_*``) and see if it is annotated with +``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser +itself proxies certain things (like some GL operations), so there is no +general way to be safe here (aside from not blocking on the main browser +thread). + +In addition, Emscripten currently has a simple model of file I/O only happening +on the main application thread (as we support JS plugin filesystems, which +cannot share memory), which is another set of operations that are proxied. + +Normally you don't need to worry about this proxying, as it is fully automatic, +however see the section on blocking right after this one. + Blocking on the main browser thread =================================== +Note that in most cases the "main browser thread" is the same as the "main +application thread". The main browser thread is literally the browser's main +thread where DOM access is allowed, and the main application thread is the one +on which you started up the application. If you started it on the main browser +thread - by it being a normal HTML page - then the two are identical. However, +you can also start a multithreaded application in a worker; in that case the +main application thread is that worker, and there is no access to the main +browser thread. + The Web API for atomics does not allow blocking on the main thread (specifically, ``Atomics.wait`` doesn't work there). Such blocking is necessary in APIs like ``pthread_join`` and anything that uses a futex wait @@ -56,27 +87,13 @@ power. (On a pthread, this isn't a problem as it runs in a Web Worker, where we don't need to busy-wait.) Busy-waiting on the main browser thread in general will work despite the -downsides just mentioned, for things like waiting on a mutex. +downsides just mentioned, for things like waiting on a lightly-contended mutex. However, things like ``pthread_join`` and ``pthread_cond_wait`` -can block for long periods of time, and if that +are often intended to block for long periods of time, and if that happens on the main browser thread, and while other threads expect it to -respond, it can cause a surprising deadlock. That can happen because some APIs -are proxied to the main browser thread because of Web limitations, and if the -main thread blocks while a worker attempts to proxy to it, a deadlock can -occur. - -In particular, DOM access is only possible on the main browser -thread, and therefore things like filesystem operations (``fopen()``, -etc.) or changing the HTML page's title, etc., are proxied over to the main -browser thread. See -`bug 3495 `_ for -more information and how to try to work around this until this. To check which -operations are proxied, you can look for the function's implementation in -the JS library (``src/library_*``) and see if it is annotated with -``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser -itself proxies certain things (like some GL operations), so there is no -general way to be safe here (aside from not blocking on the main browser -thread). +respond, it can cause a surprising deadlock. That can happen because of +proxying, see the previous section. If the main thread blocks while a worker +attempts to proxy to it, a deadlock can occur. The bottom line is that on the Web it is bad for the main browser thread to wait on anything else. Therefore by default Emscripten disallows From 7a0dc20283e3ad768bf8ade8a4d6bea255c85124 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 14 Oct 2019 14:08:11 -0700 Subject: [PATCH 16/25] support pthread_tryjoin_np --- src/library_pthread.js | 19 ++++++++++++++++--- tests/pthread/main_thread_join.cpp | 18 +++++++++++++++++- tests/test_browser.py | 3 +++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 093768e6d55ca..a039534efbf40 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -798,8 +798,8 @@ var LibraryPThread = { #endif }, - {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_block_on_main_thread'], - {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join: function(thread, status) { + _emscripten_do_pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_block_on_main_thread'], + _emscripten_do_pthread_join: function(thread, status, block) { if (!thread) { err('pthread_join attempted on a null thread pointer!'); return ERRNO_CODES.ESRCH; @@ -824,7 +824,7 @@ var LibraryPThread = { return ERRNO_CODES.EINVAL; // The thread is already detached, can no longer join it! } - if (ENVIRONMENT_IS_WEB) { + if (block && ENVIRONMENT_IS_WEB) { _emscripten_block_on_main_thread(); } @@ -839,6 +839,9 @@ var LibraryPThread = { else postMessage({ cmd: 'cleanupThread', thread: thread }); return 0; } + if (!block) { + return ERRNO_CODES.EBUSY; + } // TODO HACK! Replace the _js variant with just _pthread_testcancel: //_pthread_testcancel(); __pthread_testcancel_js(); @@ -849,6 +852,16 @@ var LibraryPThread = { } }, + {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join__deps: ['_emscripten_do_pthread_join'], + {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_join: function(thread, status) { + return __emscripten_do_pthread_join(thread, status, true); + }, + + pthread_tryjoin_np__deps: ['_emscripten_do_pthread_join'], + pthread_tryjoin_np: function(thread, status) { + return __emscripten_do_pthread_join(thread, status, false); + }, + pthread_kill__deps: ['_kill_thread'], pthread_kill: function(thread, signal) { if (signal < 0 || signal >= 65/*_NSIG*/) return ERRNO_CODES.EINVAL; diff --git a/tests/pthread/main_thread_join.cpp b/tests/pthread/main_thread_join.cpp index 7d2e8e5b07bf1..d38b9cfb3be56 100644 --- a/tests/pthread/main_thread_join.cpp +++ b/tests/pthread/main_thread_join.cpp @@ -8,6 +8,17 @@ #include #include +pthread_t thread; + +void loop() { + void* retval; + if (pthread_tryjoin_np(thread, &retval) == 0) { + emscripten_cancel_main_loop(); +#ifdef REPORT_RESULT + REPORT_RESULT(2); +#endif + } +} void *ThreadMain(void *arg) { @@ -48,13 +59,18 @@ int main() { return 0; }); + thread = CreateThread(); +#ifdef TRY_JOIN + emscripten_set_main_loop(loop, 0, 0); +#else int status; // This should fail on the main thread. puts("trying to block..."); - pthread_join(CreateThread(), (void**)&status); + pthread_join(thread, (void**)&status); puts("blocked ok."); #ifdef REPORT_RESULT REPORT_RESULT(1); #endif +#endif // TRY_JOIN } diff --git a/tests/test_browser.py b/tests/test_browser.py index 596b9f49f45b1..3cf3f51cc102a 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3622,6 +3622,9 @@ def test_pthread_main_thread_blocking(self, name, check_main, check_worker): if check_main: print('Test that without ALLOW_BLOCKING_ON_MAIN_THREAD we error on blocking on the main thread.') self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + if name == 'join': + print('Test that tryjoin is fine') + self.btest(path_from_root('tests', 'pthread', 'main_thread_join.cpp'), expected='2', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-g', '-DTRY_JOIN']) if check_worker: print('Test that everything works ok when we are on a pthread.') self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD']) From ec5c4ad6ca3a815133dcf460b9f4daeed95f2063 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 14 Oct 2019 14:17:35 -0700 Subject: [PATCH 17/25] document pthread_tryjoin_np --- site/source/docs/porting/asyncify.rst | 2 +- site/source/docs/porting/pthreads.rst | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/site/source/docs/porting/asyncify.rst b/site/source/docs/porting/asyncify.rst index 321a06c9b9104..bc55ddc26cf73 100644 --- a/site/source/docs/porting/asyncify.rst +++ b/site/source/docs/porting/asyncify.rst @@ -1,4 +1,4 @@ -.. Asyncify: +.. _Asyncify: ======================== Asyncify diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index 20729b47ec88c..e252ff2fcd2a1 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -106,6 +106,11 @@ the main browser thread to focus only on receiving proxied events. This is recommended in general, but may take some porting work, if the application assumed ``main()`` was on the main browser thread. +Another option is to replace blocking calls with nonblocking ones. For example +you can replace ``pthread_join`` with ``pthread_tryjoin_np``. This may require +your application to be refactored to use asynchronous events, perhaps through +:c:func:`emscripten_set_main_loop` or :ref:`Asyncify`. + Alternatively, you can flip a flag to allow blocking on the main browser thread, using:: From 6000e207fb9f6cb6491c7fd189ced314bd9e4f09 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 18 Oct 2019 11:06:29 -0700 Subject: [PATCH 18/25] emscripten_block_on_main_thread => emscripten_check_blocking_allowed --- src/library_pthread.js | 6 +++--- system/include/emscripten/threading.h | 2 +- system/lib/libc/musl/src/thread/pthread_cond_timedwait.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index fd1827063baaa..701e72143c02f 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -802,7 +802,7 @@ var LibraryPThread = { if (canceled == 2) throw 'Canceled!'; }, - emscripten_block_on_main_thread: function() { + emscripten_check_blocking_allowed: function() { #if ASSERTIONS assert(ENVIRONMENT_IS_WEB); #endif @@ -811,7 +811,7 @@ var LibraryPThread = { #endif }, - _emscripten_do_pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_block_on_main_thread'], + _emscripten_do_pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_check_blocking_allowed'], _emscripten_do_pthread_join: function(thread, status, block) { if (!thread) { err('pthread_join attempted on a null thread pointer!'); @@ -838,7 +838,7 @@ var LibraryPThread = { } if (block && ENVIRONMENT_IS_WEB) { - _emscripten_block_on_main_thread(); + _emscripten_check_blocking_allowed(); } for (;;) { diff --git a/system/include/emscripten/threading.h b/system/include/emscripten/threading.h index 3d66e5f739545..b4dee0d016640 100644 --- a/system/include/emscripten/threading.h +++ b/system/include/emscripten/threading.h @@ -374,7 +374,7 @@ struct thread_profiler_block // Called when blocking on the main thread. This will error if main thread // blocking is not enabled, see ALLOW_BLOCKING_ON_MAIN_THREAD. -void emscripten_block_on_main_thread(void); +void emscripten_check_blocking_allowed(void); #ifdef __cplusplus } diff --git a/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c b/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c index 407b2a8f73dfc..ccb49f92dce07 100644 --- a/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c +++ b/system/lib/libc/musl/src/thread/pthread_cond_timedwait.c @@ -82,7 +82,7 @@ int __pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restri #ifdef __EMSCRIPTEN__ if (pthread_self() == emscripten_main_browser_thread_id()) { - emscripten_block_on_main_thread(); + emscripten_check_blocking_allowed(); } #endif From 3529e3f56071acbe284a1ade7d2654bbd5633da2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 18 Oct 2019 14:36:15 -0700 Subject: [PATCH 19/25] mention pthread_tryjoin_np in changelog [ci skip] --- ChangeLog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index f61b79a0270ec..d550f40f4e071 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -21,6 +21,8 @@ Current Trunk it, you must set `ALLOW_BLOCKING_ON_MAIN_THREAD` (disallowing by default makes it less likely new users of pthreads will hit surprising deadlocks on the Web). + - Add `pthread_tryjoin_np`, which is a POSIX API similar to `pthread_join` + but without blocking. - Add support for overriding `.emscripten` config variables using environment variables. Any config variable `FOO` can be overridden by `EM_FOO` in the environment. From fa2882fd81eba9fc1afa72c23c352a740e9f3429 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 28 Oct 2019 13:10:33 -0700 Subject: [PATCH 20/25] just warn for now --- ChangeLog.md | 6 +- site/source/docs/porting/pthreads.rst | 17 ++-- src/library_pthread.js | 1 + src/settings.js | 6 +- tests/test_browser.py | 110 +++++++++++++------------- 5 files changed, 67 insertions(+), 73 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index e4129ba6b226c..7c5ddc83ccbdc 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,10 +17,8 @@ See docs/process.md for how version tagging works. Current Trunk ------------- - - Pthread blocking on the main thread is now disallowed by default. To enable - it, you must set `ALLOW_BLOCKING_ON_MAIN_THREAD` (disallowing by default - makes it less likely new users of pthreads will hit surprising deadlocks - on the Web). + - Pthread blocking on the main thread will now warn in the console. If + `ALLOW_BLOCKING_ON_MAIN_THREAD` is unset then the warning is an error. - Add `pthread_tryjoin_np`, which is a POSIX API similar to `pthread_join` but without blocking. - Only MEMFS is included by default, others (NODEFS, IDBFS, WORKERFS, PROXYFS) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index e252ff2fcd2a1..04e3ffee005cf 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -96,11 +96,12 @@ proxying, see the previous section. If the main thread blocks while a worker attempts to proxy to it, a deadlock can occur. The bottom line is that on the Web it is bad for the main browser thread to -wait on anything else. Therefore by default Emscripten disallows -``pthread_join`` and ``pthread_cond_wait`` on the main browser thread, and -will throw an error (whose message will point to here). +wait on anything else. Therefore by default Emscripten warns if +``pthread_join`` and ``pthread_cond_wait`` happen on the main browser thread, +and will throw an error if ``ALLOW_BLOCKING_ON_MAIN_THREAD`` is zero +(whose message will point to here). -To avoid this problem, you can use ``PROXY_TO_PTHREAD``, which as +To avoid these problems, you can use ``PROXY_TO_PTHREAD``, which as mentioned earlier moves your ``main()`` function to a pthread, which leaves the main browser thread to focus only on receiving proxied events. This is recommended in general, but may take some porting work, if the application @@ -111,14 +112,6 @@ you can replace ``pthread_join`` with ``pthread_tryjoin_np``. This may require your application to be refactored to use asynchronous events, perhaps through :c:func:`emscripten_set_main_loop` or :ref:`Asyncify`. -Alternatively, you can flip a flag to allow blocking on the main browser thread, -using:: - - -s ALLOW_BLOCKING_ON_MAIN_THREAD=1 - -This lets you avoid the error, but you have the risk of deadlocks and other -problems as mentioned earlier. - Special considerations ====================== diff --git a/src/library_pthread.js b/src/library_pthread.js index 701e72143c02f..40da916f757c7 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -805,6 +805,7 @@ var LibraryPThread = { emscripten_check_blocking_allowed: function() { #if ASSERTIONS assert(ENVIRONMENT_IS_WEB); + warnOnce('Blocking on the main thread is very dangerous, see https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); #endif #if !ALLOW_BLOCKING_ON_MAIN_THREAD abort('Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); diff --git a/src/settings.js b/src/settings.js index 8142366deb87e..d8df1bcc9d46c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1305,11 +1305,13 @@ var PTHREAD_HINT_NUM_CORES = 4; // True when building with --threadprofiler var PTHREADS_PROFILING = 0; -// By default we disallow running pthread_join and pthread_cond_wait +// It is dangerous to call pthread_join or pthread_cond_wait // on the main thread, as doing so can cause deadlocks on the Web (and also // it works using a busy-wait which is expensive). See // https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread -var ALLOW_BLOCKING_ON_MAIN_THREAD = 0; +// This may become set to 0 by default in the future; for now, this just +// warns in the console. +var ALLOW_BLOCKING_ON_MAIN_THREAD = 1; // If true, add in debug traces for diagnosing pthreads related issues. var PTHREADS_DEBUG = 0; diff --git a/tests/test_browser.py b/tests/test_browser.py index f0c33ec4bb76d..3b5cd8c491646 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1916,7 +1916,7 @@ def test_gl_glteximage(self): @requires_graphics_hardware @requires_threads def test_gl_textures(self): - for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']]: + for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1']]: self.btest('gl_textures.cpp', '0', args=['-lGL'] + args) @requires_graphics_hardware @@ -3606,12 +3606,12 @@ def prep_no_SAB(self): }) @requires_threads def test_pthread_atomics(self, args=[]): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-g1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + args) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-g1'] + args) # Test 64-bit atomics. @requires_threads def test_pthread_64bit_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test 64-bit C++11 atomics. @requires_threads @@ -3621,20 +3621,20 @@ def test_pthread_64bit_cxx11_atomics(self): self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_cxx11_atomics.cpp'), expected='0', args=opt + pthreads + ['-std=c++11']) @parameterized({ - 'join': ['join', True, True], - 'wait': ['wait', True, False], + 'join': ['join',], + 'wait': ['wait',], }) @requires_threads - def test_pthread_main_thread_blocking(self, name, check_main, check_worker): - if check_main: - print('Test that without ALLOW_BLOCKING_ON_MAIN_THREAD we error on blocking on the main thread.') - self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) - if name == 'join': - print('Test that tryjoin is fine') - self.btest(path_from_root('tests', 'pthread', 'main_thread_join.cpp'), expected='2', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-g', '-DTRY_JOIN']) - if check_worker: + def test_pthread_main_thread_blocking(self, name): + print('Test that we error if not ALLOW_BLOCKING_ON_MAIN_THREAD') + self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) + if name == 'join': + print('Test that by default we just warn about blocking on the main thread.') + self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) + print('Test that tryjoin is fine, even if not ALLOW_BLOCKING_ON_MAIN_THREAD') + self.btest(path_from_root('tests', 'pthread', 'main_thread_join.cpp'), expected='2', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-g', '-DTRY_JOIN', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) print('Test that everything works ok when we are on a pthread.') - self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD']) + self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) # Test the old GCC atomic __sync_fetch_and_op builtin operations. @requires_threads @@ -3648,40 +3648,40 @@ def test_pthread_gcc_atomic_fetch_and_op(self): print(args) if self.is_wasm_backend() and '--separate-asm' in args or 'AGGRESSIVE_VARIABLE_ELIMINATION=1' in args: continue - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_fetch_and_op.cpp'), expected='0', args=args + ['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_fetch_and_op.cpp'), expected='0', args=args + ['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # 64 bit version of the above test. @requires_threads def test_pthread_gcc_64bit_atomic_fetch_and_op(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_fetch_and_op.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_fetch_and_op.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) # Test the old GCC atomic __sync_op_and_fetch builtin operations. @requires_threads def test_pthread_gcc_atomic_op_and_fetch(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) # 64 bit version of the above test. @requires_threads def test_pthread_gcc_64bit_atomic_op_and_fetch(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_64bit_atomic_op_and_fetch.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'], also_asmjs=True) # Tests the rest of the remaining GCC atomics after the two above tests. @requires_threads def test_pthread_gcc_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test the __sync_lock_test_and_set and __sync_lock_release primitives. @requires_threads def test_pthread_gcc_spinlock(self): for arg in [[], ['-DUSE_EMSCRIPTEN_INTRINSICS']]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_spinlock.cpp'), expected='800', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + arg, also_asmjs=True) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_gcc_spinlock.cpp'), expected='800', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + arg, also_asmjs=True) # Test that basic thread creation works. @requires_threads def test_pthread_create(self): def test(args): print(args) - self.btest(path_from_root('tests', 'pthread', 'test_pthread_create.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + args) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_create.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + args) test([]) test(['-O3']) @@ -3695,7 +3695,7 @@ def test_pthread_preallocates_workers(self): # Test that allocating a lot of threads doesn't regress. This needs to be checked manually! @requires_threads def test_pthread_large_pthread_allocation(self): - self.btest(path_from_root('tests', 'pthread', 'test_large_pthread_allocation.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=128MB', '-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=50', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], message='Check output from test to ensure that a regression in time it takes to allocate the threads has not occurred.') + self.btest(path_from_root('tests', 'pthread', 'test_large_pthread_allocation.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=128MB', '-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=50'], message='Check output from test to ensure that a regression in time it takes to allocate the threads has not occurred.') # Tests the -s PROXY_TO_PTHREAD=1 option. @requires_threads @@ -3706,17 +3706,17 @@ def test_pthread_proxy_to_pthread(self): @requires_threads def test_pthread_create_pthread(self): for modularize in [[], ['-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', path_from_root('tests', 'shell_that_launches_modularize.html')]]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_create_pthread.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + modularize) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_create_pthread.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2'] + modularize) # Test another case of pthreads spawning pthreads, but this time the callers immediately join on the threads they created. @requires_threads def test_pthread_nested_spawns(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_nested_spawns.cpp'), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2']) # Test that main thread can wait for a pthread to finish via pthread_join(). @requires_threads def test_pthread_join(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_join.cpp'), expected='6765', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_join.cpp'), expected='6765', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test that threads can rejoin the pool once detached and finished @requires_threads @@ -3737,42 +3737,42 @@ def test_pthread_kill(self): # Test that pthread cleanup stack (pthread_cleanup_push/_pop) works. @requires_threads def test_pthread_cleanup(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_cleanup.cpp'), expected='907640832', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_cleanup.cpp'), expected='907640832', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Tests the pthread mutex api. @requires_threads def test_pthread_mutex(self): for arg in [[], ['-DSPINLOCK_TEST']]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + arg) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + arg) @requires_threads def test_pthread_attr_getstack(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_attr_getstack.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_attr_getstack.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2']) # Test that memory allocation is thread-safe. @requires_threads def test_pthread_malloc(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Stress test pthreads allocating memory that will call to sbrk(), and main thread has to free up the data. @requires_threads def test_pthread_malloc_free(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=256MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=256MB']) # Test that the pthread_barrier API works ok. @requires_threads def test_pthread_barrier(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_barrier.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_barrier.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test the pthread_once() function. @requires_threads def test_pthread_once(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_once.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_once.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test against a certain thread exit time handling bug by spawning tons of threads. @requires_threads def test_pthread_spawns(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_spawns.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_spawns.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # It is common for code to flip volatile global vars for thread control. This is a bit lax, but nevertheless, test whether that # kind of scheme will work with Emscripten as well. @@ -3784,18 +3784,18 @@ def test_pthread_volatile(self): # Test thread-specific data (TLS). @requires_threads def test_pthread_thread_local_storage(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_thread_local_storage.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ASSERTIONS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_thread_local_storage.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ASSERTIONS=1']) # Test the pthread condition variable creation and waiting. @requires_threads def test_pthread_condition_variable(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_condition_variable.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_condition_variable.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) # Test that pthreads are able to do printf. @requires_threads def test_pthread_printf(self): def run(debug): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_printf.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'LIBRARY_DEBUG=%d' % debug, '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_printf.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'LIBRARY_DEBUG=%d' % debug]) run(debug=True) run(debug=False) @@ -3803,7 +3803,7 @@ def run(debug): # Test that pthreads are able to do cout. Failed due to https://bugzilla.mozilla.org/show_bug.cgi?id=1154858. @requires_threads def test_pthread_iostream(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_iostream.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_iostream.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test that the main thread is able to use pthread_set/getspecific. @requires_threads @@ -3818,12 +3818,12 @@ def test_pthread_num_logical_cores(self): # Test that pthreads have access to filesystem. @requires_threads def test_pthread_file_io(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_file_io.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_file_io.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test that the pthread_create() function operates benignly in the case that threading is not supported. @requires_threads def test_pthread_supported(self): - for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']]: + for args in [[], ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']]: self.btest(path_from_root('tests', 'pthread', 'test_pthread_supported.cpp'), expected='0', args=['-O3'] + args) # Test that --separate-asm works with -s USE_PTHREADS=1. @@ -3831,7 +3831,7 @@ def test_pthread_supported(self): @requires_threads def test_pthread_separate_asm_pthreads(self): for modularize in [[], ['-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', path_from_root('tests', 'shell_that_launches_modularize.html')]]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '--separate-asm', '--profiling'] + modularize) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '--profiling'] + modularize) # Test the operation of Module.pthreadMainPrefixURL variable @no_wasm_backend('uses js') @@ -3865,7 +3865,7 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile" string to locate where worker.js will be loaded from. create_test_file('shell.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-o', 'test.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-o', 'test.html']) shutil.move('test.worker.js', os.path.join('cdn', 'test.worker.js')) if self.is_wasm_backend(): shutil.copyfile('test.html.mem', os.path.join('cdn', 'test.html.mem')) @@ -3873,14 +3873,14 @@ def test_pthread_custom_pthread_main_url(self): # Test that it is possible to define "Module.locateFile(foo)" function to locate where worker.js will be loaded from. create_test_file('shell2.html', open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function(filename) { if (filename == "test.worker.js") return "cdn/test.worker.js"; else return filename; }, ')) - self.compile_btest(['main.cpp', '--shell-file', 'shell2.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-o', 'test2.html']) + self.compile_btest(['main.cpp', '--shell-file', 'shell2.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-o', 'test2.html']) try_delete('test.worker.js') self.run_browser('test2.html', '', '/report_result?1') # Test that if the main thread is performing a futex wait while a pthread needs it to do a proxied operation (before that pthread would wake up the main thread), that it's not a deadlock. @requires_threads def test_pthread_proxying_in_futex_wait(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_proxying_in_futex_wait.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_proxying_in_futex_wait.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test that sbrk() operates properly in multithreaded conditions @requires_threads @@ -3889,7 +3889,7 @@ def test_pthread_sbrk(self): print('aborting malloc=' + str(aborting_malloc)) # With aborting malloc = 1, test allocating memory in threads # With aborting malloc = 0, allocate so much memory in threads that some of the allocations fail. - self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=128MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=128MB']) # Test that -s ABORTING_MALLOC=0 works in both pthreads and non-pthreads builds. (sbrk fails gracefully) @requires_threads @@ -3901,12 +3901,12 @@ def test_pthread_gauge_available_memory(self): # Test that the proxying operations of user code from pthreads to main thread work @requires_threads def test_pthread_run_on_main_thread(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test how a lot of back-to-back called proxying operations behave. @requires_threads def test_pthread_run_on_main_thread_flood(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread_flood.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_run_on_main_thread_flood.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test that it is possible to synchronously call a JavaScript function on the main thread and get a return value back. @requires_threads @@ -3943,7 +3943,7 @@ def test_pthread_clock_drift(self): @requires_threads def test_pthread_utf8_funcs(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_utf8_funcs.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_utf8_funcs.cpp'), expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) # Test the emscripten_futex_wake(addr, INT_MAX); functionality to wake all waiters @requires_threads @@ -4499,7 +4499,7 @@ def test_asmfs_relative_paths(self): def test_pthread_locale(self): for args in [ [], - ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'], + ['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2'], ]: print("Testing with: ", args) self.btest('pthread/test_pthread_locale.c', expected='1', args=args) @@ -4526,10 +4526,10 @@ def test_pthread_run_script(self): def test_emscripten_animate_canvas_element_size(self): for args in [ ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1'], - ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], - ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_EXPLICIT_CONTEXT_SWAP=1'], - ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], - ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_MANUALLY_SET_ELEMENT_CSS_SIZE=1'], + ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], + ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_EXPLICIT_CONTEXT_SWAP=1'], + ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1'], + ['-DTEST_EXPLICIT_CONTEXT_SWAP=1', '-s', 'PROXY_TO_PTHREAD=1', '-s', 'USE_PTHREADS=1', '-s', 'OFFSCREEN_FRAMEBUFFER=1', '-DTEST_MANUALLY_SET_ELEMENT_CSS_SIZE=1'], ['-DTEST_EMSCRIPTEN_SET_MAIN_LOOP=1', '-s', 'OFFSCREENCANVAS_SUPPORT'], ]: cmd = ['-lGL', '-O3', '-g2', '--shell-file', path_from_root('tests', 'canvas_animate_resize_shell.html'), '--separate-asm', '-s', 'GL_DEBUG=1', '--threadprofiler'] + args @@ -4548,7 +4548,7 @@ def test_pthread_hello_thread(self): @requires_threads def test_pthread_growth_mainthread(self): def run(emcc_args=[]): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth_mainthread.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + emcc_args, also_asmjs=False) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth_mainthread.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB'] + emcc_args, also_asmjs=False) run() run(['-s', 'MODULARIZE_INSTANCE=1']) @@ -4559,7 +4559,7 @@ def run(emcc_args=[]): @requires_threads def test_pthread_growth(self): def run(emcc_args=[]): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-g', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD'] + emcc_args, also_asmjs=False) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_memory_growth.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=2', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=32MB', '-s', 'WASM_MEM_MAX=256MB', '-g'] + emcc_args, also_asmjs=False) run() run(['-s', 'ASSERTIONS=1']) @@ -4680,7 +4680,7 @@ def test_unicode_html_shell(self): # Tests the functionality of the emscripten_thread_sleep() function. @requires_threads def test_emscripten_thread_sleep(self): - self.btest(path_from_root('tests', 'pthread', 'emscripten_thread_sleep.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["print"]', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD']) + self.btest(path_from_root('tests', 'pthread', 'emscripten_thread_sleep.c'), expected='1', args=['-s', 'USE_PTHREADS=1', '-s', 'EXTRA_EXPORTED_RUNTIME_METHODS=["print"]']) # Tests that Emscripten-compiled applications can be run from a relative path in browser that is different than the address of the current page def test_browser_run_from_different_directory(self): From 3158218728532436c3b0cf530959c45fba98ab36 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 28 Oct 2019 17:30:06 -0700 Subject: [PATCH 21/25] flake8 --- tests/test_browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 3b5cd8c491646..df90038988ed9 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3621,8 +3621,8 @@ def test_pthread_64bit_cxx11_atomics(self): self.btest(path_from_root('tests', 'pthread', 'test_pthread_64bit_cxx11_atomics.cpp'), expected='0', args=opt + pthreads + ['-std=c++11']) @parameterized({ - 'join': ['join',], - 'wait': ['wait',], + 'join': ('join',), + 'wait': ('wait',), }) @requires_threads def test_pthread_main_thread_blocking(self, name): From a0ebf2f5ca723788aa2eda0cb7793a3f14074f01 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 30 Oct 2019 13:32:07 -0700 Subject: [PATCH 22/25] feedback [ci skip] --- site/source/docs/porting/pthreads.rst | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index 04e3ffee005cf..4d6e0f8b7ac93 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -50,7 +50,7 @@ The Web allows certain operations to only happen from the main browser thread, like interacting with the DOM. As a result, various operations are proxied to the main browser thread if they are called on a background thread. See `bug 3495 `_ for -more information and how to try to work around this until this. To check which +more information and how to try to work around this until then. To check which operations are proxied, you can look for the function's implementation in the JS library (``src/library_*``) and see if it is annotated with ``__proxy: 'sync'`` or ``__proxy: 'async'``; however, note that the browser @@ -60,22 +60,23 @@ thread). In addition, Emscripten currently has a simple model of file I/O only happening on the main application thread (as we support JS plugin filesystems, which -cannot share memory), which is another set of operations that are proxied. +cannot share memory); this is another set of operations that are proxied. -Normally you don't need to worry about this proxying, as it is fully automatic, -however see the section on blocking right after this one. +Proxying can cause problems in certain cases, see the section on blocking right +after this one. Blocking on the main browser thread =================================== Note that in most cases the "main browser thread" is the same as the "main -application thread". The main browser thread is literally the browser's main -thread where DOM access is allowed, and the main application thread is the one -on which you started up the application. If you started it on the main browser -thread - by it being a normal HTML page - then the two are identical. However, -you can also start a multithreaded application in a worker; in that case the -main application thread is that worker, and there is no access to the main -browser thread. +application thread". The main browser thread is where web pages start to run +JavaScript, and where JavaScript can access the DOM (a page can also create a Web +Worker, which would no longer be on the main thread). The main application +thread is the one on which you started up the Emscripten JavaScript file. If you +started it on the main browser thread - by it being a normal HTML page - then +the two are identical. However, you can also start a multithreaded application +in a worker; in that case the main application thread is that worker, and there +is no access to the main browser thread. The Web API for atomics does not allow blocking on the main thread (specifically, ``Atomics.wait`` doesn't work there). Such blocking is From b3bfc83f4cdbc378ce5ec3b405bcadfb78cd4ad8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 30 Oct 2019 13:34:53 -0700 Subject: [PATCH 23/25] tryjoin without a pool --- tests/test_browser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_browser.py b/tests/test_browser.py index df90038988ed9..87b8ecc209768 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3633,6 +3633,8 @@ def test_pthread_main_thread_blocking(self, name): self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1']) print('Test that tryjoin is fine, even if not ALLOW_BLOCKING_ON_MAIN_THREAD') self.btest(path_from_root('tests', 'pthread', 'main_thread_join.cpp'), expected='2', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-g', '-DTRY_JOIN', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) + print('Test that tryjoin is fine, even if not ALLOW_BLOCKING_ON_MAIN_THREAD, and even without a pool') + self.btest(path_from_root('tests', 'pthread', 'main_thread_join.cpp'), expected='2', args=['-O3', '-s', 'USE_PTHREADS=1', '-g', '-DTRY_JOIN', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) print('Test that everything works ok when we are on a pthread.') self.btest(path_from_root('tests', 'pthread', 'main_thread_%s.cpp' % name), expected='1', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-s', 'PROXY_TO_PTHREAD', '-s', 'ALLOW_BLOCKING_ON_MAIN_THREAD=0']) From 345e0e7d93b54cae45f477dc337c43230e9acada Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 30 Oct 2019 13:43:00 -0700 Subject: [PATCH 24/25] Make sure we test failed tryjoins --- tests/pthread/main_thread_join.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/pthread/main_thread_join.cpp b/tests/pthread/main_thread_join.cpp index d38b9cfb3be56..951adb34aa89a 100644 --- a/tests/pthread/main_thread_join.cpp +++ b/tests/pthread/main_thread_join.cpp @@ -8,25 +8,37 @@ #include #include +#include + pthread_t thread; +std::atomic tries; + +static const int EXPECTED_TRIES = 7; + void loop() { void* retval; + printf("try...\n"); if (pthread_tryjoin_np(thread, &retval) == 0) { emscripten_cancel_main_loop(); + assert(tries.load() == EXPECTED_TRIES); #ifdef REPORT_RESULT REPORT_RESULT(2); #endif } + tries++; } -void *ThreadMain(void *arg) -{ +void *ThreadMain(void *arg) { +#ifdef TRY_JOIN + // Delay to force the main thread to try and fail a few times before + // succeeding. + while (tries.load() < EXPECTED_TRIES) {} +#endif pthread_exit((void*)0); } -pthread_t CreateThread() -{ +pthread_t CreateThread() { pthread_t ret; int rc = pthread_create(&ret, NULL, ThreadMain, (void*)0); assert(rc == 0); @@ -34,8 +46,7 @@ pthread_t CreateThread() } int main() { - if (!emscripten_has_threading_support()) - { + if (!emscripten_has_threading_support()) { #ifdef REPORT_RESULT REPORT_RESULT(0); #endif From 2bc7ce493b5337c0d1aab3a95578da2071211dfd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 30 Oct 2019 13:53:31 -0700 Subject: [PATCH 25/25] text nit [ci skip] --- site/source/docs/porting/pthreads.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst index 4d6e0f8b7ac93..a1c3524422dfb 100644 --- a/site/source/docs/porting/pthreads.rst +++ b/site/source/docs/porting/pthreads.rst @@ -62,8 +62,7 @@ In addition, Emscripten currently has a simple model of file I/O only happening on the main application thread (as we support JS plugin filesystems, which cannot share memory); this is another set of operations that are proxied. -Proxying can cause problems in certain cases, see the section on blocking right -after this one. +Proxying can cause problems in certain cases, see the section on blocking below. Blocking on the main browser thread ===================================