diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c846dfa0279c..d752eb2ec62b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1102,6 +1102,9 @@ jobs: test-browser-firefox: executor: ubuntu-lts environment: + # Deserializing a growable SharedArrayBuffer is currently broken in Firefox. + # See: https://github.com/emscripten-core/emscripten/issues/27118 + # See: https://bugzilla.mozilla.org/show_bug.cgi?id=2021136 EMTEST_LACKS_GROWABLE_ARRAYBUFFERS: "1" steps: - prepare-for-tests diff --git a/ChangeLog.md b/ChangeLog.md index ac78148d3d2bf..3bafc1d912120 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -52,6 +52,9 @@ See docs/process.md for more on how version tagging works. run dependencies). This means that errors during startup (or during the `main()` function) will more often show up as unhandled promise rejections (`onunhandledreject`) rather than synchronous errors (`onerror`). (#27121) +- The `GROWABLE_ARRAYBUFFERS` setting now support both `=1` (auto-detect and + use the feature) and `=2` (unconditionally use the feature). The second mode + is still useful for avoiding the overhead in multi-threaded builds. (#27096) 6.0.0 - 06/04/26 ---------------- diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index 7bdf76ebf4dca..cbe6b367b57e4 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -3373,11 +3373,16 @@ Default value: false GROWABLE_ARRAYBUFFERS ===================== -Enable support for GrowableSharedArrayBuffer. -This feature has only recently become available across major browser engines -and Node.js. +Enable support for growable views of Wasm memory. This is a recent Web +platform feature that can make growing the Wasm memory more efficient, +especially in multi-threaded builds. +Setting this to 1 will auto-detect the presence of this API and use it +when available. +Setting this to 2 will unconditionally require it. This is the only way +to completely remove the overhead of growable memory + pthreads. +This settings does nothing unless ALLOW_MEMORY_GROWTH is set. -Default value: false +Default value: 0 .. _cross_origin: diff --git a/src/audio_worklet.js b/src/audio_worklet.js index 2745b542afe05..8eb686e15a440 100644 --- a/src/audio_worklet.js +++ b/src/audio_worklet.js @@ -96,9 +96,8 @@ function createWasmAudioWorkletProcessor() { process(inputList, outputList) { #endif -#if ALLOW_MEMORY_GROWTH +#if ALLOW_MEMORY_GROWTH && GROWABLE_ARRAYBUFFERS != 2 // Recreate the output views if the heap has changed - // TODO: add support for GROWABLE_ARRAYBUFFERS if (HEAPF32.buffer != this.outputViews[0].buffer) { this.createOutputViews(); } diff --git a/src/lib/libcore.js b/src/lib/libcore.js index c27396fb30e37..5e8265111c251 100644 --- a/src/lib/libcore.js +++ b/src/lib/libcore.js @@ -217,7 +217,7 @@ addToLibrary({ try { // round size grow request up to wasm page size (fixed 64KB per spec) wasmMemory.grow({{{ toIndexType('pages') }}}); // .grow() takes a delta compared to the previous size -#if !GROWABLE_ARRAYBUFFERS +#if GROWABLE_ARRAYBUFFERS != 2 updateMemoryViews(); #endif #if MEMORYPROFILER @@ -357,7 +357,7 @@ addToLibrary({ #endif // ALLOW_MEMORY_GROWTH }, -#if !GROWABLE_ARRAYBUFFERS +#if GROWABLE_ARRAYBUFFERS != 2 // Called after wasm grows memory. At that time we need to update the views. // Without this notification, we'd need to check the buffer in JS every time // we return from any wasm, which adds overhead. See @@ -2515,7 +2515,7 @@ function wrapSyscallFunction(x, library, isWasi) { library[x + '__deps'] ??= []; -#if PURE_WASI && !GROWABLE_ARRAYBUFFERS +#if PURE_WASI && GROWABLE_ARRAYBUFFERS != 2 // In PURE_WASI mode we can't assume the wasm binary was built by emscripten // and politely notify us on memory growth. Instead we have to check for // possible memory growth on each syscall. diff --git a/src/runtime_common.js b/src/runtime_common.js index aea1cefcef55d..d50b9bce7db02 100644 --- a/src/runtime_common.js +++ b/src/runtime_common.js @@ -12,7 +12,7 @@ #include "runtime_safe_heap.js" #endif -#if SHARED_MEMORY && ALLOW_MEMORY_GROWTH && !GROWABLE_ARRAYBUFFERS +#if SHARED_MEMORY && ALLOW_MEMORY_GROWTH && GROWABLE_ARRAYBUFFERS != 2 // Support for growable heap + pthreads, where the buffer may change, so JS views // must be updated. function growMemViews() { @@ -108,17 +108,57 @@ var runtimeExited = false; }; }}} + +#if ALLOW_MEMORY_GROWTH +// When ALLOW_MEMORY_GROWTH is enabled, the conversion from Wasm +// memory to ArrayBuffer requires some additional logic. +function getMemoryBuffer() { +#if GROWABLE_ARRAYBUFFERS == 2 + return wasmMemory.toResizableBuffer(); +#else +#if GROWABLE_ARRAYBUFFERS == 1 +#if SHARED_MEMORY && (MIN_FIREFOX_VERSION != TARGET_NOT_SUPPORTED) + // Deserializing a growable SharedArrayBuffer is currently broken in Firefox. + // See: https://github.com/emscripten-core/emscripten/issues/27118 + // See: https://bugzilla.mozilla.org/show_bug.cgi?id=2021136 + if (!globalThis.navigator?.userAgent?.match(/firefox/i)) { +#endif + try { + // This method may be missing or could fail with `Memory must have a maximum` + var b = wasmMemory.toResizableBuffer(); +#if SHARED_MEMORY + growMemViews = () => {}; +#endif + return b; + + } catch {} +#if SHARED_MEMORY && (MIN_FIREFOX_VERSION != TARGET_NOT_SUPPORTED) + } +#endif +#endif // GROWABLE_ARRAYBUFFERS == 1 + return wasmMemory.buffer; +#endif // GROWABLE_ARRAYBUFFERS == 2 +} +#endif // ALLOW_MEMORY_GROWTH + function updateMemoryViews() { #if RUNTIME_DEBUG dbg(`updateMemoryViews: first=${!HEAP8} size=${wasmMemory.buffer.byteLength}`); #endif -#if !ALLOW_MEMORY_GROWTH && ASSERTIONS +#if ALLOW_MEMORY_GROWTH + // If we already have a heap that is resizeable/growable buffer we don't + // need to do anything in updateMemoryViews. +#if SHARED_MEMORY + if (HEAP8?.buffer?.growable) return; +#else + if (HEAP8?.buffer?.resizable) return; +#endif + var b = getMemoryBuffer(); +#else +#if ASSERTIONS // When memory growth is disabled this function should be called exactly once. assert(!HEAP8, 'updateMemoryViews should only be called once when ALLOW_MEMORY_GROWTH=0'); #endif -#if GROWABLE_ARRAYBUFFERS - var b = wasmMemory.toResizableBuffer(); -#else var b = wasmMemory.buffer; #endif {{{ maybeExportHeap('HEAP8') }}}HEAP8 = new Int8Array(b); diff --git a/src/settings.js b/src/settings.js index 74ac3c75b84e0..bbad7cfe0f83c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2220,11 +2220,16 @@ var WASM_ESM_INTEGRATION = false; // [link] var JS_BASE64_API = false; -// Enable support for GrowableSharedArrayBuffer. -// This feature has only recently become available across major browser engines -// and Node.js. -// [link] -var GROWABLE_ARRAYBUFFERS = false; +// Enable support for growable views of Wasm memory. This is a recent Web +// platform feature that can make growing the Wasm memory more efficient, +// especially in multi-threaded builds. +// Setting this to 1 will auto-detect the presence of this API and use it +// when available. +// Setting this to 2 will unconditionally require it. This is the only way +// to completely remove the overhead of growable memory + pthreads. +// This settings does nothing unless ALLOW_MEMORY_GROWTH is set. +// [link] +var GROWABLE_ARRAYBUFFERS = 0; // If the emscripten-generated program is hosted on separate origin then // starting new pthread worker can violate CSP rules. Enabling diff --git a/test/codesize/test_codesize_mem_O3_grow.json b/test/codesize/test_codesize_mem_O3_grow.json index 57771a52cc8ab..3e74d065b3e74 100644 --- a/test/codesize/test_codesize_mem_O3_grow.json +++ b/test/codesize/test_codesize_mem_O3_grow.json @@ -1,10 +1,10 @@ { - "a.out.js": 4526, - "a.out.js.gz": 2185, + "a.out.js": 4551, + "a.out.js.gz": 2197, "a.out.nodebug.wasm": 5261, "a.out.nodebug.wasm.gz": 2419, - "total": 9787, - "total_gz": 4604, + "total": 9812, + "total_gz": 4616, "sent": [ "a (emscripten_resize_heap)" ], diff --git a/test/codesize/test_codesize_mem_O3_grow_standalone.json b/test/codesize/test_codesize_mem_O3_grow_standalone.json index 3cb910c8e7f86..3cd6ed62252d6 100644 --- a/test/codesize/test_codesize_mem_O3_grow_standalone.json +++ b/test/codesize/test_codesize_mem_O3_grow_standalone.json @@ -1,10 +1,10 @@ { - "a.out.js": 3995, - "a.out.js.gz": 1927, + "a.out.js": 4014, + "a.out.js.gz": 1935, "a.out.nodebug.wasm": 5641, "a.out.nodebug.wasm.gz": 2659, - "total": 9636, - "total_gz": 4586, + "total": 9655, + "total_gz": 4594, "sent": [ "args_get", "args_sizes_get", diff --git a/test/codesize/test_codesize_minimal_pthreads_memgrowth.json b/test/codesize/test_codesize_minimal_pthreads_memgrowth.json index d7c9da9b3ec65..c21cf87dbe2fd 100644 --- a/test/codesize/test_codesize_minimal_pthreads_memgrowth.json +++ b/test/codesize/test_codesize_minimal_pthreads_memgrowth.json @@ -1,10 +1,10 @@ { - "a.out.js": 7362, - "a.out.js.gz": 3631, + "a.out.js": 7381, + "a.out.js.gz": 3643, "a.out.nodebug.wasm": 19064, "a.out.nodebug.wasm.gz": 8804, - "total": 26426, - "total_gz": 12435, + "total": 26445, + "total_gz": 12447, "sent": [ "a (memory)", "b (exit)", diff --git a/test/embind/embind_test.cpp b/test/embind/embind_test.cpp index a0862f653cfe6..1c8ca6e937419 100644 --- a/test/embind/embind_test.cpp +++ b/test/embind/embind_test.cpp @@ -228,10 +228,27 @@ void force_memory_growth() { assert(val::global("oldheap")["byteLength"].as() == old_size); emscripten_resize_heap(old_size + EMSCRIPTEN_PAGE_SIZE); assert(emscripten_get_heap_size() > old_size); - // HEAP8 on the module should now be rebound, and our oldheap should be - // detached + // HEAP8 on the module should always be correct after the resize. + // Our oldheap reference may be detached, depending on whether the + // buffer is resizable. assert(val::module_property("HEAP8")["byteLength"].as() > old_size); - assert(val::global("oldheap")["byteLength"].as() == 0); + + val oldheap = val::global("oldheap"); + val buffer = oldheap["buffer"]; + bool growable = false; + if (!buffer.isUndefined()) { + if (!buffer["resizable"].isUndefined()) { + growable = buffer["resizable"].as(); + } else if (!buffer["growable"].isUndefined()) { + growable = buffer["growable"].as(); + } + } + + if (growable) { + assert(oldheap["byteLength"].as() == emscripten_get_heap_size()); + } else { + assert(oldheap["byteLength"].as() == 0); + } } std::string emval_test_take_and_return_const_char_star(const char* str) { diff --git a/test/test_browser.py b/test/test_browser.py index 66c57f788b9cc..3afa261a6fb4d 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -4622,21 +4622,21 @@ def test_minimal_runtime_hello_thread(self, opts): # Tests memory growth in pthreads mode, but still on the main thread. @parameterized({ '': ([], 1), - 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'], 1), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS=2', '-Wno-experimental'], 1), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'], 2), }) @no_highmem('uses INITIAL_MEMORY') @requires_growable_arraybuffers def test_pthread_growth_mainthread(self, cflags, pthread_pool_size): self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - if '-sGROWABLE_ARRAYBUFFERS' not in cflags: + if '-sGROWABLE_ARRAYBUFFERS=2' not in cflags: self.cflags.append('-Wno-pthreads-mem-growth') self.btest_exit('pthread/test_pthread_memory_growth_mainthread.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) # Tests memory growth in a pthread. @parameterized({ '': ([],), - 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'],), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS=2', '-Wno-experimental'],), 'assert': (['-sASSERTIONS'],), 'proxy': (['-sPROXY_TO_PTHREAD'], 2), 'minimal': (['-sMINIMAL_RUNTIME', '-sMODULARIZE', '-sEXPORT_NAME=MyModule'],), @@ -4645,7 +4645,7 @@ def test_pthread_growth_mainthread(self, cflags, pthread_pool_size): @requires_growable_arraybuffers def test_pthread_growth(self, cflags, pthread_pool_size=1): self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - if '-sGROWABLE_ARRAYBUFFERS' not in cflags: + if '-sGROWABLE_ARRAYBUFFERS=2' not in cflags: self.cflags.append('-Wno-pthreads-mem-growth') self.btest_exit('pthread/test_pthread_memory_growth.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) diff --git a/test/test_other.py b/test/test_other.py index cdaa5ef5b5aee..fa798c2c5f000 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13253,11 +13253,11 @@ def test_pthread_sigmask(self, args): @requires_pthreads @parameterized({ '': ([],), - 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'],), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS=2', '-Wno-experimental'],), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'],), }) def test_pthread_growth_mainthread(self, cflags): - if '-sGROWABLE_ARRAYBUFFERS' in cflags: + if '-sGROWABLE_ARRAYBUFFERS=2' in cflags: self.require_node_26() else: self.cflags.append('-Wno-pthreads-mem-growth') @@ -13270,7 +13270,7 @@ def test_pthread_join_interrupted(self): @requires_node_26 def test_growable_arraybuffers(self): self.do_runf('hello_world.c', - cflags=['-O2', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'], + cflags=['-O2', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sGROWABLE_ARRAYBUFFERS=2', '-Wno-experimental'], output_basename='growable') self.do_runf('hello_world.c', cflags=['-O2', '-pthread', '-sALLOW_MEMORY_GROWTH', '-Wno-pthreads-mem-growth'], @@ -13284,7 +13284,7 @@ def test_growable_arraybuffers(self): @requires_pthreads @parameterized({ '': ([],), - 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'],), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS=2', '-Wno-experimental'],), 'assert': (['-sASSERTIONS'],), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'],), 'minimal': (['-sMINIMAL_RUNTIME', '-sMODULARIZE', '-sEXPORT_NAME=MyModule'],), @@ -13295,7 +13295,7 @@ def test_pthread_growth(self, cflags): # TODO: Switch this to a "require Node.js 24" check self.require_node_25() - if '-sGROWABLE_ARRAYBUFFERS' in cflags: + if '-sGROWABLE_ARRAYBUFFERS=2' in cflags: self.require_node_26() else: self.cflags.append('-Wno-pthreads-mem-growth') diff --git a/tools/building.py b/tools/building.py index ce88d3031d3c5..ea95f2e7d66ba 100644 --- a/tools/building.py +++ b/tools/building.py @@ -1079,7 +1079,7 @@ def little_endian_heap(js_file): def apply_wasm_memory_growth(js_file): - assert not settings.GROWABLE_ARRAYBUFFERS + assert settings.GROWABLE_ARRAYBUFFERS != 2 logger.debug('supporting wasm memory growth with pthreads') return acorn_optimizer(js_file, ['growableHeap']) diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index f5558148681d0..a26e3c8ae375b 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -281,7 +281,7 @@ def apply_min_browser_versions(): enable_feature(Feature.WASM_LEGACY_EXCEPTIONS, 'Wasm Legacy exceptions (-fwasm-exceptions with -sWASM_LEGACY_EXCEPTIONS=1)') else: enable_feature(Feature.WASM_EXCEPTIONS, 'Wasm exceptions (-fwasm-exceptions with -sWASM_LEGACY_EXCEPTIONS=0)') - if settings.GROWABLE_ARRAYBUFFERS: + if settings.GROWABLE_ARRAYBUFFERS == 2: enable_feature(Feature.GROWABLE_ARRAYBUFFERS, 'GrowableSharedArrayBuffer') @@ -290,4 +290,5 @@ def auto_enable_features(): # TODO(sbc): Find make a generic way to expose the feature matrix to JS # compiler rather then adding them all ad-hoc as internal settings default_setting('WASM_BIGINT', caniuse(Feature.JS_BIGINT_INTEGRATION)) - default_setting('GROWABLE_ARRAYBUFFERS', caniuse(Feature.GROWABLE_ARRAYBUFFERS)) + if caniuse(Feature.GROWABLE_ARRAYBUFFERS): + default_setting('GROWABLE_ARRAYBUFFERS', 2) diff --git a/tools/link.py b/tools/link.py index 746d530fc38e2..0b633c110864b 100644 --- a/tools/link.py +++ b/tools/link.py @@ -490,7 +490,7 @@ def setup_pthreads(): # pthreads + dynamic linking has certain limitations if settings.MAIN_MODULE: diagnostics.warning('experimental', 'dynamic linking + pthreads is experimental') - if settings.ALLOW_MEMORY_GROWTH and not settings.GROWABLE_ARRAYBUFFERS: + if settings.ALLOW_MEMORY_GROWTH and settings.GROWABLE_ARRAYBUFFERS != 2: diagnostics.warning('pthreads-mem-growth', '-pthread + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') default_setting('DEFAULT_PTHREAD_STACK_SIZE', settings.STACK_SIZE) @@ -2334,7 +2334,7 @@ def phase_binaryen(target, options, wasm_target): # unsigning pass. # we also must do this after the asan or safe_heap instrumentation, as they # wouldn't be able to recognize patterns produced by the growth pass. - if settings.SHARED_MEMORY and settings.ALLOW_MEMORY_GROWTH and not settings.GROWABLE_ARRAYBUFFERS: + if settings.SHARED_MEMORY and settings.ALLOW_MEMORY_GROWTH and settings.GROWABLE_ARRAYBUFFERS != 2: with ToolchainProfiler.profile_block('apply_wasm_memory_growth'): final_js = building.apply_wasm_memory_growth(final_js) diff --git a/tools/system_libs.py b/tools/system_libs.py index d133928437021..b041e8a90d212 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2252,7 +2252,7 @@ def vary_on(cls): def get_default_variation(cls, **kwargs): return super().get_default_variation( is_mem_grow=settings.ALLOW_MEMORY_GROWTH, - is_pure=settings.PURE_WASI or settings.GROWABLE_ARRAYBUFFERS, + is_pure=settings.PURE_WASI or settings.GROWABLE_ARRAYBUFFERS == 2, nocatch=settings.DISABLE_EXCEPTION_CATCHING and not settings.WASM_EXCEPTIONS, **kwargs, )