From d486dc187d3dffe91cdf26e784e1054240383a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 6 Dec 2019 18:40:47 +0200 Subject: [PATCH 01/27] Add pthreads support to MINIMAL_RUNTIME. --- emcc.py | 13 ++-- src/library.js | 12 ++++ src/library_pthread.js | 24 +++++-- src/modules.js | 20 ++++-- src/parseTools.js | 10 +++ src/postamble_minimal.js | 25 ++++++- src/preamble_minimal.js | 49 ++++++++++--- src/settings.js | 6 ++ src/shell_minimal.js | 32 ++++++--- src/shell_minimal_runtime.html | 6 +- src/worker.js | 82 ++++++++++++++++++---- tests/pthread/call_async_on_main_thread.js | 2 +- tests/pthread/test_pthread_run_script.cpp | 12 ++-- tools/minimal_runtime_shell.py | 11 ++- 14 files changed, 248 insertions(+), 56 deletions(-) diff --git a/emcc.py b/emcc.py index d83fbf9f49a9a..a5aad7639453c 100755 --- a/emcc.py +++ b/emcc.py @@ -1538,6 +1538,9 @@ def is_supported_link_flag(f): # consider it a user export, i.e., one which can never be removed). shared.Building.user_requested_exports += ['dynCall_ii'] + if shared.Settings.MINIMAL_RUNTIME: + shared.Building.user_requested_exports += ['exit'] + if shared.Settings.PROXY_TO_PTHREAD: shared.Settings.EXPORTED_FUNCTIONS += ['_proxy_main'] @@ -1554,7 +1557,12 @@ def include_and_export(name): # can access them: # general threading variables: - shared.Settings.EXPORTED_RUNTIME_METHODS += ['PThread', 'ExitStatus'] + shared.Settings.EXPORTED_RUNTIME_METHODS += ['PThread'] + + # To keep code size to minimum, MINIMAL_RUNTIME does not utilize the global ExitStatus + # object, only regular runtime has it. + if not shared.Settings.MINIMAL_RUNTIME: + shared.Settings.EXPORTED_RUNTIME_METHODS += ['ExitStatus'] # stack check: if shared.Settings.STACK_OVERFLOW_CHECK: @@ -1921,9 +1929,6 @@ def check_human_readable_list(items): if shared.Settings.EMTERPRETIFY: exit_with_error('-s EMTERPRETIFY=1 is not supported with -s MINIMAL_RUNTIME=1') - if shared.Settings.USE_PTHREADS: - exit_with_error('-s USE_PTHREADS=1 is not yet supported with -s MINIMAL_RUNTIME=1') - if shared.Settings.PRECISE_F32 == 2: exit_with_error('-s PRECISE_F32=2 is not supported with -s MINIMAL_RUNTIME=1') diff --git a/src/library.js b/src/library.js index 8c89ff1170873..8903f55c27533 100644 --- a/src/library.js +++ b/src/library.js @@ -275,6 +275,12 @@ LibraryManager.library = { _Exit__sig: 'vi', _Exit: 'exit', +#if MINIMAL_RUNTIME + $exit: function(status) { + throw 'exit(' + status + ')'; + }, +#endif + fork__deps: ['__setErrNo'], fork: function() { // pid_t fork(void); @@ -1357,10 +1363,16 @@ LibraryManager.library = { $establishStackSpace__asm: true, $establishStackSpace__sig: 'vii', $establishStackSpace: function(stackBase, stackMax) { +#if WASM_BACKEND + // The wasm backend path does not have a way to set the stack max, so ignore + // the stack max parameter, this function only resets the stack base. + stackRestore(stackBase); +#else stackBase = stackBase|0; stackMax = stackMax|0; STACKTOP = stackBase; STACK_MAX = stackMax; +#endif }, #endif diff --git a/src/library_pthread.js b/src/library_pthread.js index 74e3a655bc779..a20276efd0ff6 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -389,7 +389,11 @@ var LibraryPThread = { 'urlOrBlob': Module['mainScriptUrlOrBlob'] || _scriptDir, #if WASM 'wasmMemory': wasmMemory, - 'wasmModule': wasmModule, +#if MINIMAL_RUNTIME + wasmModule: Module['wasm'], +#else + wasmModule: wasmModule, +#endif #if LOAD_SOURCE_MAP 'wasmSourceMap': wasmSourceMap, #endif @@ -400,17 +404,23 @@ var LibraryPThread = { 'buffer': HEAPU8.buffer, 'asmJsUrlOrBlob': Module["asmJsUrlOrBlob"], #endif +#if !MINIMAL_RUNTIME 'DYNAMIC_BASE': DYNAMIC_BASE, +#endif 'DYNAMICTOP_PTR': DYNAMICTOP_PTR }); }, // Creates a new web Worker and places it in the unused worker pool to wait for its use. allocateUnusedWorker: function() { +#if MINIMAL_RUNTIME + var pthreadMainJs = Module['worker']; +#else // Allow HTML module to configure the location where the 'worker.js' file will be loaded from, // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative // to the main html file is loaded. var pthreadMainJs = locateFile('{{{ PTHREAD_WORKER_FILE }}}'); +#endif #if PTHREADS_DEBUG out('Allocating a new web worker from ' + pthreadMainJs); #endif @@ -769,17 +779,21 @@ var LibraryPThread = { if (canceled == 2) throw 'Canceled!'; }, - emscripten_check_blocking_allowed: function() { #if ASSERTIONS + emscripten_check_blocking_allowed: function() { assert(ENVIRONMENT_IS_WEB); warnOnce('Blocking on the main thread is very dangerous, see https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); -#endif #if !ALLOW_BLOCKING_ON_MAIN_THREAD abort('Blocking on the main thread is not allowed by default. See https://emscripten.org/docs/porting/pthreads.html#blocking-on-the-main-browser-thread'); #endif }, +#endif - _emscripten_do_pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', 'emscripten_check_blocking_allowed'], + _emscripten_do_pthread_join__deps: ['_cleanup_thread', '_pthread_testcancel_js', 'emscripten_main_thread_process_queued_calls', 'emscripten_futex_wait', +#if ASSERTIONS + 'emscripten_check_blocking_allowed' +#endif + ], _emscripten_do_pthread_join: function(thread, status, block) { if (!thread) { err('pthread_join attempted on a null thread pointer!'); @@ -805,9 +819,11 @@ var LibraryPThread = { return ERRNO_CODES.EINVAL; // The thread is already detached, can no longer join it! } +#if ASSERTIONS if (block && ENVIRONMENT_IS_WEB) { _emscripten_check_blocking_allowed(); } +#endif for (;;) { var threadStatus = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.threadStatus }}} ) >> 2); diff --git a/src/modules.js b/src/modules.js index f7a35003263ff..0f7caf645a9ce 100644 --- a/src/modules.js +++ b/src/modules.js @@ -87,6 +87,10 @@ var LibraryManager = { libraries.push('library_browser.js'); } + if (USE_PTHREADS) { // TODO: Currently WebGL proxying makes pthreads library depend on WebGL. + libraries.push('library_webgl.js'); + } + if (FILESYSTEM) { // Core filesystem libraries (always linked against, unless -s FILESYSTEM=0 is specified) libraries = libraries.concat([ @@ -472,10 +476,13 @@ function exportRuntime() { // In pthreads mode, the following functions always need to be exported to // Module for closure compiler, and also for MODULARIZE (so worker.js can // access them). - var threadExports = ['PThread', 'ExitStatus', '_pthread_self']; + var threadExports = ['PThread', '_pthread_self']; if (WASM) { threadExports.push('wasmMemory'); } + if (!MINIMAL_RUNTIME) { + threadExports.push('ExitStatus'); + } threadExports.forEach(function(x) { EXPORTED_RUNTIME_METHODS_SET[x] = 1; @@ -512,11 +519,16 @@ function exportRuntime() { } } } - return runtimeElements.map(function(name) { + function removeEmptyElements(x) { + return x.filter(function(e) { + return e != null; + }) + } + return removeEmptyElements(runtimeElements.map(function(name) { return maybeExport(name); - }).join('\n') + runtimeNumbers.map(function(name) { + })).join('\n') + removeEmptyElements(runtimeNumbers.map(function(name) { return maybeExportNumber(name); - }).join('\n'); + })).join('\n'); } var PassManager = { diff --git a/src/parseTools.js b/src/parseTools.js index 9c9ead903e57f..12d8acad0c24e 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1658,3 +1658,13 @@ function buildStringArray(array) { return '[]'; } } + +// Generates access to a JS global scope variable in pthreads worker.js. In MODULARIZE mode the JS scope is not directly accessible, so all the relevant variables +// are exported via Module. In non-MODULARIZE mode, we can directly access the variables in global scope. +function makeAsmGlobalAccessInPthread(variable) { + if (MODULARIZE || !MINIMAL_RUNTIME) { + return "Module['" + variable + "']"; // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case. + } else { + return variable; + } +} diff --git a/src/postamble_minimal.js b/src/postamble_minimal.js index c004eff1c6f41..b6bb17e0161c4 100644 --- a/src/postamble_minimal.js +++ b/src/postamble_minimal.js @@ -118,6 +118,10 @@ WebAssembly.instantiateStreaming(fetch('{{{ TARGET_BASENAME }}}.wasm'), imports) // Module['wasm'] should contain a typed array of the Wasm object data, or a precompiled WebAssembly Module. if (!Module['wasm']) throw 'Must load WebAssembly Module in to variable Module.wasm before adding compiled output .js script to the DOM'; #endif +#if USE_PTHREADS +// Store the Wasm instance to be able to post it to pthreads. +Module['wasmInstance'] = +#endif WebAssembly.instantiate(Module['wasm'], imports).then(function(output) { #endif @@ -132,13 +136,15 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) { // output object will have an output.instance and output.module objects. But if Module['wasm'] // is an already compiled WebAssembly module, then output is the WebAssembly instance itself. // Depending on the build mode, Module['wasm'] can mean a different thing. -#if MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION || MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION +#if MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION || MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION || USE_PTHREADS // https://caniuse.com/#feat=wasm and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming // Firefox 52 added Wasm support, but only Firefox 58 added compileStreaming & instantiateStreaming. // Chrome 57 added Wasm support, but only Chrome 61 added compileStreaming & instantiateStreaming. // Node.js and Safari do not support compileStreaming or instantiateStreaming. -#if MIN_FIREFOX_VERSION < 58 || MIN_CHROME_VERSION < 61 || ENVIRONMENT_MAY_BE_NODE || MIN_SAFARI_VERSION != TARGET_NOT_SUPPORTED - asm = output.instance ? output.instance.exports : output.exports; +#if MIN_FIREFOX_VERSION < 58 || MIN_CHROME_VERSION < 61 || ENVIRONMENT_MAY_BE_NODE || MIN_SAFARI_VERSION != TARGET_NOT_SUPPORTED || USE_PTHREADS + // In pthreads, Module['wasm'] is an already compiled WebAssembly.Module. In that case, 'output' is a WebAssembly.Instance. + // In main thread, Module['wasm'] is either a typed array or a fetch stream. In that case, 'output.instance' is the WebAssembly.Instance. + var asm = (output.instance || output).exports; #else asm = output.exports; #endif @@ -153,8 +159,14 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) { #else /*** ASM_MODULE_EXPORTS ***/ #endif + initRuntime(asm); +#if PTHREAD_POOL_SIZE + if (!ENVIRONMENT_IS_PTHREAD) PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, ready); + else ready(); +#else ready(); +#endif }) #if ASSERTIONS .catch(function(error) { @@ -167,8 +179,15 @@ WebAssembly.instantiate(Module['wasm'], imports).then(function(output) { // Initialize asm.js (synchronous) initRuntime(asm); + +#if PTHREAD_POOL_SIZE +if (!ENVIRONMENT_IS_PTHREAD) PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, ready); +else ready(); +#else ready(); +#endif #endif {{GLOBAL_VARS}} + diff --git a/src/preamble_minimal.js b/src/preamble_minimal.js index 2b2f5b4ac53df..66254379f1152 100644 --- a/src/preamble_minimal.js +++ b/src/preamble_minimal.js @@ -12,7 +12,11 @@ function assert(condition, text) { #endif function abort(what) { +#if ASSERTIONS + throw new Error(what); +#else throw what; +#endif } var tempRet0 = 0; @@ -50,12 +54,16 @@ Module['wasm'] = base64Decode('{{{ getQuoted("WASM_BINARY_DATA") }}}'); #include "runtime_sab_polyfill.js" #if USE_PTHREADS +var STATIC_BASE = {{{ GLOBAL_BASE }}}; + if (!ENVIRONMENT_IS_PTHREAD) { #endif var GLOBAL_BASE = {{{ GLOBAL_BASE }}}, TOTAL_STACK = {{{ TOTAL_STACK }}}, +#if !USE_PTHREADS STATIC_BASE = {{{ GLOBAL_BASE }}}, +#endif STACK_BASE = {{{ getQuoted('STACK_BASE') }}}, STACKTOP = STACK_BASE, STACK_MAX = {{{ getQuoted('STACK_MAX') }}} @@ -82,6 +90,15 @@ var wasmMemory = new WebAssembly.Memory({ #endif }); +var buffer = wasmMemory.buffer; + +#if USE_PTHREADS +} +#if ASSERTIONS +assert(buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag'); +#endif +#endif + var wasmTable = new WebAssembly.Table({ 'initial': {{{ getQuoted('WASM_TABLE_SIZE') }}}, #if !ALLOW_TABLE_GROWTH @@ -94,12 +111,6 @@ var wasmTable = new WebAssembly.Table({ 'element': 'anyfunc' }); -var buffer = wasmMemory.buffer; - -#if USE_PTHREADS && ASSERTIONS -assert(buffer instanceof SharedArrayBuffer, 'requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag'); -#endif - #else #if USE_PTHREADS @@ -116,14 +127,20 @@ var buffer = new ArrayBuffer({{{ TOTAL_MEMORY }}}); #if ASSERTIONS var WASM_PAGE_SIZE = 65536; -assert(STACK_BASE % 16 === 0, 'stack must start aligned'); -assert(({{{ getQuoted('DYNAMIC_BASE') }}}) % 16 === 0, 'heap must start aligned'); +#if USE_PTHREADS +if (!ENVIRONMENT_IS_PTHREAD) { +#endif +assert(STACK_BASE % 16 === 0, 'stack must start aligned to 16 bytes, STACK_BASE==' + STACK_BASE); +assert(({{{ getQuoted('DYNAMIC_BASE') }}}) % 16 === 0, 'heap must start aligned to 16 bytes, DYNAMIC_BASE==' + {{{ getQuoted('DYNAMIC_BASE') }}}); assert({{{ TOTAL_MEMORY }}} >= TOTAL_STACK, 'TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + {{{ TOTAL_MEMORY }}} + '! (TOTAL_STACK=' + TOTAL_STACK + ')'); assert({{{ TOTAL_MEMORY }}} % WASM_PAGE_SIZE === 0); #if WASM_MEM_MAX != -1 assert({{{ WASM_MEM_MAX }}} % WASM_PAGE_SIZE == 0); #endif assert(buffer.byteLength === {{{ TOTAL_MEMORY }}}); +#if USE_PTHREADS +} +#endif #endif // ASSERTIONS #if ALLOW_MEMORY_GROWTH @@ -157,10 +174,20 @@ var HEAPF64 = new Float64Array(buffer); #endif #if MEM_INIT_METHOD == 1 && !MEM_INIT_IN_WASM && !SINGLE_FILE + +#if USE_PTHREADS +if (!ENVIRONMENT_IS_PTHREAD) { +#endif + #if ASSERTIONS if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM'; #endif HEAPU8.set(new Uint8Array(Module['mem']), GLOBAL_BASE); + +#if USE_PTHREADS +} +#endif + #endif #if SINGLE_FILE && !WASM && !WASM_BACKEND @@ -169,7 +196,11 @@ HEAPU8.set(base64Decode('{{{ getQuoted("BASE64_MEMORY_INITIALIZER") }}}'), GLOBA #endif #if USES_DYNAMIC_ALLOC -HEAP32[DYNAMICTOP_PTR>>2] = {{{ getQuoted('DYNAMIC_BASE') }}}; + HEAP32[DYNAMICTOP_PTR>>2] = {{{ getQuoted('DYNAMIC_BASE') }}}; +#endif + +#if USE_PTHREADS +} #endif #include "runtime_stack_check.js" diff --git a/src/settings.js b/src/settings.js index badeb2e65b8e3..c0c36cd0dfce1 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1291,6 +1291,12 @@ var WASM_MEM_MAX = -1; // (This option was formerly called BINARYEN_ASYNC_COMPILATION) var WASM_ASYNC_COMPILATION = 1; +// If set to 1, MINIMAL_RUNTIME will utilize streaming WebAssembly compilation +// (via WebAssembly.instantiateStreaming). This requires the MIME type of the +// application to be set correctly. If set to 0, non-streaming compilation +// (via WebAssembly.instantiate) will be used. +var STREAMING_WASM_COMPILATION = 1; + // WebAssembly defines a "producers section" which compilers and tools can // annotate themselves in. Emscripten does not emit this by default, as it // increases code size, and some users may not want information about their tools diff --git a/src/shell_minimal.js b/src/shell_minimal.js index 1c9935181455d..f159d34810641 100644 --- a/src/shell_minimal.js +++ b/src/shell_minimal.js @@ -21,6 +21,18 @@ var ENVIRONMENT_IS_NODE = typeof process === 'object'; var ENVIRONMENT_IS_SHELL = typeof read === 'function'; #endif +#if ASSERTIONS +#if !ENVIRONMENT_MAY_BE_NODE && !ENVIRONMENT_MAY_BE_SHELL +var ENVIRONMENT_IS_WEB = true +#else +#if ENVIRONMENT && ENVIRONMENT.indexOf(',') < 0 +var ENVIRONMENT_IS_WEB = {{{ ENVIRONMENT === 'web' }}}; +#else +var ENVIRONMENT_IS_WEB = !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_SHELL; +#endif +#endif +#endif + #if ASSERTIONS && ENVIRONMENT_MAY_BE_NODE && ENVIRONMENT_MAY_BE_SHELL if (ENVIRONMENT_IS_NODE && ENVIRONMENT_IS_SHELL) { throw 'unclear environment'; @@ -117,7 +129,13 @@ function err(text) { // compilation is ready. In that callback, call the function run() to start // the program. function ready() { - run(); +#if USE_PTHREADS + if (!ENVIRONMENT_IS_PTHREAD) { +#endif + run(); +#if USE_PTHREADS + } +#endif } // --pre-jses are emitted after the Module integration code, so that they can @@ -133,16 +151,12 @@ function ready() { var _scriptDir = (typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined; #endif -var ENVIRONMENT_IS_PTHREAD; -if (!ENVIRONMENT_IS_PTHREAD) ENVIRONMENT_IS_PTHREAD = false; // ENVIRONMENT_IS_PTHREAD=true will have been preset in pthread-main.js. Make it false in the main runtime thread. +// MINIMAL_RUNTIME does not support --proxy-to-worker option, so Worker and Pthread environments +// coincide. +var ENVIRONMENT_IS_WORKER = ENVIRONMENT_IS_PTHREAD = typeof importScripts === 'function'; -if (typeof ENVIRONMENT_IS_PTHREAD === 'undefined') { - // ENVIRONMENT_IS_PTHREAD=true will have been preset in pthread-main.js. Make it false in the main runtime thread. - // N.B. this line needs to appear without 'var' keyword to avoid 'var hoisting' from occurring. (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/var) - ENVIRONMENT_IS_PTHREAD = false; -} #if MODULARIZE -else { +if (ENVIRONMENT_IS_WORKER) { var buffer = {{{EXPORT_NAME}}}.buffer; var STATICTOP = {{{EXPORT_NAME}}}.STATICTOP; var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}.DYNAMICTOP_PTR; diff --git a/src/shell_minimal_runtime.html b/src/shell_minimal_runtime.html index a42604bc73a31..711102b5ecef9 100644 --- a/src/shell_minimal_runtime.html +++ b/src/shell_minimal_runtime.html @@ -3,7 +3,11 @@