From 6c05f32397d4b424cc7075f553ff236eb635eea4 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:15:48 -0400 Subject: [PATCH 01/20] stdout in wasmfs --- src/modules.js | 11 ++++++-- src/preamble.js | 2 +- src/settings.js | 4 +++ system/lib/fetch/wasmfs.cpp | 55 +++++++++++++++++++++++++++++++++++++ tools/system_libs.py | 5 ++++ 5 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 system/lib/fetch/wasmfs.cpp diff --git a/src/modules.js b/src/modules.js index a1167ba9bf44d..b35a32f701c89 100644 --- a/src/modules.js +++ b/src/modules.js @@ -61,14 +61,17 @@ var LibraryManager = { 'library_formatString.js', 'library_math.js', 'library_path.js', - 'library_syscall.js', 'library_html5.js', 'library_stack_trace.js', - 'library_wasi.js', 'library_int53.js', 'library_dylink.js' ]; + if (!WASMFS) { + libraries.push('library_syscall.js') + libraries.push('library_wasi.js') + } + if (LINK_AS_CXX && !EXCEPTION_HANDLING) { if (DISABLE_EXCEPTION_THROWING) { libraries.push('library_exceptions_stub.js'); @@ -93,7 +96,9 @@ var LibraryManager = { libraries.push('library_html5_webgl.js'); } - if (FILESYSTEM) { + if (WASMFS) { + libraries.push('library_wasmfs.js') + } else if (FILESYSTEM) { // Core filesystem libraries (always linked against, unless -s FILESYSTEM=0 is specified) libraries = libraries.concat([ 'library_fs.js', diff --git a/src/preamble.js b/src/preamble.js index 4fba1b2cb6940..38574e61cdf39 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -617,7 +617,7 @@ function abort(what) { #include "memoryprofiler.js" -#if ASSERTIONS && !('$FS' in addedLibraryItems) && !ASMFS +#if ASSERTIONS && !('$FS' in addedLibraryItems) && !ASMFS && !WASMFS // show errors on likely calls to FS when it was not included var FS = { error: function() { diff --git a/src/settings.js b/src/settings.js index fd3d02b7ee17d..b9045c3672495 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1635,6 +1635,10 @@ var FETCH = 0; // [link] var ASMFS = 0; +// If set to 1, uses new filesystem implementation +// [link] +var WASMFS = 0; + // If set to 1, embeds all subresources in the emitted file as base64 string // literals. Embedded subresources may include (but aren't limited to) wasm, // asm.js, and static memory initialization code. diff --git a/system/lib/fetch/wasmfs.cpp b/system/lib/fetch/wasmfs.cpp new file mode 100644 index 0000000000000..2a5aa15f02c4f --- /dev/null +++ b/system/lib/fetch/wasmfs.cpp @@ -0,0 +1,55 @@ +// // Copyright 2021 The Emscripten Authors. All rights reserved. +// // Emscripten is available under two separate licenses, the MIT license and the +// // University of Illinois/NCSA Open Source License. Both these licenses can be +// // found in the LICENSE file. + +#include +#define __NEED_struct_iovec +#include +#include + +extern "C" { + + __wasi_errno_t __wasi_fd_write( __wasi_fd_t fd, const __wasi_ciovec_t *iovs, size_t iovs_len, __wasi_size_t *nwritten) { + if (fd == 1 || fd == 2) { + MAIN_THREAD_EM_ASM({ + var buffer = []; + var fd = $0; + var iovs = $1; + var iovs_len = $2; + var nwritten = $3; + var num = 0; + for (var i = 0; i < iovs_len; i++) { + var ptr = HEAP32[(((iovs)+(i*8))>>2)]; + var len = HEAP32[(((iovs)+(i*8 + 4))>>2)]; + for (var j = 0; j < len; j++) { + if (HEAPU8[ptr+j] === 0 || HEAPU8[ptr+j] === 10) { + (fd === 1 ? out : err)(UTF8ArrayToString(buffer, 0)); + buffer.length = 0; + } else { + buffer.push(HEAPU8[ptr+j]); + } + } + num += len; + } + HEAP32[((nwritten)>>2)] = num; + }, fd, iovs, iovs_len, nwritten); + } + return 0; + } + + __wasi_errno_t __wasi_fd_seek(__wasi_fd_t fd, __wasi_filedelta_t offset, __wasi_whence_t whence, __wasi_filesize_t *newoffset) + { + abort(); + } + + __wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) + { + abort(); + } + + __wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, size_t iovs_len, __wasi_size_t *nread) + { + abort(); + } +} \ No newline at end of file diff --git a/tools/system_libs.py b/tools/system_libs.py index d59180a5e7b9e..c033ff3991060 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1283,6 +1283,11 @@ def can_build(self): # https://github.com/emscripten-core/emscripten/issues/9534 return True +class libwasmfs(MTLibrary): + name = 'libwasmfs' + + def get_files(self): + return [utils.path_from_root('system/lib/fetch/wasmfs.cpp')] class libhtml5(Library): name = 'libhtml5' From 7b7069093f13c2d4c0f5076241783e5a16cdf456 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:20:32 -0400 Subject: [PATCH 02/20] emcc updated --- emcc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/emcc.py b/emcc.py index 88fd76a85b5b6..8a128e25fc001 100755 --- a/emcc.py +++ b/emcc.py @@ -1767,6 +1767,12 @@ def default_setting(name, new_default): settings.FETCH = 1 settings.JS_LIBRARIES.append((0, 'library_asmfs.js')) + if settings.WASMFS: + state.forced_stdlibs.append('libwasmfs') + settings.FILESYSTEM = 0 + settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 + # settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) + # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. if not settings.USES_DYNAMIC_ALLOC: settings.MALLOC = 'none' From 6939f12e9f4f7abed1da3441a57ca126e6eef24a Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:22:54 -0400 Subject: [PATCH 03/20] add library_wasmfs --- src/library_wasmfs.js | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/library_wasmfs.js diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js new file mode 100644 index 0000000000000..49cae0303a892 --- /dev/null +++ b/src/library_wasmfs.js @@ -0,0 +1,8 @@ +var WASMFS = { + $WASMFS : {} + +} + +mergeInto(LibraryManager.library, WASMFS); + +DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$WASMFS'); \ No newline at end of file From f2ce6b426e42527308ca4efe53d388f73069708e Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:26:59 -0400 Subject: [PATCH 04/20] add library_wasmfs to emcc.py --- emcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 8a128e25fc001..1ed0e4801cc47 100755 --- a/emcc.py +++ b/emcc.py @@ -1771,7 +1771,7 @@ def default_setting(name, new_default): state.forced_stdlibs.append('libwasmfs') settings.FILESYSTEM = 0 settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 - # settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) + settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. if not settings.USES_DYNAMIC_ALLOC: From d1b1e81c6dcef4f2e85860f9956bb749544261aa Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:28:57 -0400 Subject: [PATCH 05/20] whitespace --- system/lib/fetch/wasmfs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/fetch/wasmfs.cpp b/system/lib/fetch/wasmfs.cpp index 2a5aa15f02c4f..5264af60a7937 100644 --- a/system/lib/fetch/wasmfs.cpp +++ b/system/lib/fetch/wasmfs.cpp @@ -52,4 +52,4 @@ extern "C" { { abort(); } -} \ No newline at end of file +} From f26043d69b5901d5ef81bd69cd110f25b7c5283d Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:30:10 -0400 Subject: [PATCH 06/20] remove default list --- src/library_wasmfs.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index 49cae0303a892..afdfa4db60990 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -4,5 +4,3 @@ var WASMFS = { } mergeInto(LibraryManager.library, WASMFS); - -DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$WASMFS'); \ No newline at end of file From 84d4bc5d2bf0b4a46a121418958036d5c5e984e3 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Wed, 22 Sep 2021 19:35:14 -0400 Subject: [PATCH 07/20] add blank lines --- tools/system_libs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/system_libs.py b/tools/system_libs.py index c033ff3991060..add54a4012247 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1283,12 +1283,14 @@ def can_build(self): # https://github.com/emscripten-core/emscripten/issues/9534 return True + class libwasmfs(MTLibrary): name = 'libwasmfs' def get_files(self): return [utils.path_from_root('system/lib/fetch/wasmfs.cpp')] + class libhtml5(Library): name = 'libhtml5' From 92d5304f69c4a952f8457d3be286c360d6e1254a Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Thu, 23 Sep 2021 11:13:45 -0400 Subject: [PATCH 08/20] addressed comments --- emcc.py | 2 +- src/library_wasmfs.js | 6 --- src/modules.js | 11 ++-- src/settings.js | 2 + system/lib/fetch/wasmfs.cpp | 55 -------------------- system/lib/wasmfs/wasmfs.cpp | 72 ++++++++++++++++++++++++++ tests/core/test_wasmfs_hello_world.c | 14 +++++ tests/core/test_wasmfs_hello_world.out | 1 + tests/test_core.py | 11 ++++ tools/system_libs.py | 2 +- 10 files changed, 105 insertions(+), 71 deletions(-) delete mode 100644 src/library_wasmfs.js delete mode 100644 system/lib/fetch/wasmfs.cpp create mode 100644 system/lib/wasmfs/wasmfs.cpp create mode 100644 tests/core/test_wasmfs_hello_world.c create mode 100644 tests/core/test_wasmfs_hello_world.out diff --git a/emcc.py b/emcc.py index 1ed0e4801cc47..9c6e02601b17e 100755 --- a/emcc.py +++ b/emcc.py @@ -1771,7 +1771,7 @@ def default_setting(name, new_default): state.forced_stdlibs.append('libwasmfs') settings.FILESYSTEM = 0 settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 - settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) + # settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) TODO: Add in future PR # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. if not settings.USES_DYNAMIC_ALLOC: diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js deleted file mode 100644 index afdfa4db60990..0000000000000 --- a/src/library_wasmfs.js +++ /dev/null @@ -1,6 +0,0 @@ -var WASMFS = { - $WASMFS : {} - -} - -mergeInto(LibraryManager.library, WASMFS); diff --git a/src/modules.js b/src/modules.js index b35a32f701c89..a1167ba9bf44d 100644 --- a/src/modules.js +++ b/src/modules.js @@ -61,17 +61,14 @@ var LibraryManager = { 'library_formatString.js', 'library_math.js', 'library_path.js', + 'library_syscall.js', 'library_html5.js', 'library_stack_trace.js', + 'library_wasi.js', 'library_int53.js', 'library_dylink.js' ]; - if (!WASMFS) { - libraries.push('library_syscall.js') - libraries.push('library_wasi.js') - } - if (LINK_AS_CXX && !EXCEPTION_HANDLING) { if (DISABLE_EXCEPTION_THROWING) { libraries.push('library_exceptions_stub.js'); @@ -96,9 +93,7 @@ var LibraryManager = { libraries.push('library_html5_webgl.js'); } - if (WASMFS) { - libraries.push('library_wasmfs.js') - } else if (FILESYSTEM) { + if (FILESYSTEM) { // Core filesystem libraries (always linked against, unless -s FILESYSTEM=0 is specified) libraries = libraries.concat([ 'library_fs.js', diff --git a/src/settings.js b/src/settings.js index b9045c3672495..2d6b12d94428d 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1635,6 +1635,8 @@ var FETCH = 0; // [link] var ASMFS = 0; +// ATTENTION [WIP]: Experimental feature, use at your own risk +// Will eventually replace current JS file system implementation // If set to 1, uses new filesystem implementation // [link] var WASMFS = 0; diff --git a/system/lib/fetch/wasmfs.cpp b/system/lib/fetch/wasmfs.cpp deleted file mode 100644 index 5264af60a7937..0000000000000 --- a/system/lib/fetch/wasmfs.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// // Copyright 2021 The Emscripten Authors. All rights reserved. -// // Emscripten is available under two separate licenses, the MIT license and the -// // University of Illinois/NCSA Open Source License. Both these licenses can be -// // found in the LICENSE file. - -#include -#define __NEED_struct_iovec -#include -#include - -extern "C" { - - __wasi_errno_t __wasi_fd_write( __wasi_fd_t fd, const __wasi_ciovec_t *iovs, size_t iovs_len, __wasi_size_t *nwritten) { - if (fd == 1 || fd == 2) { - MAIN_THREAD_EM_ASM({ - var buffer = []; - var fd = $0; - var iovs = $1; - var iovs_len = $2; - var nwritten = $3; - var num = 0; - for (var i = 0; i < iovs_len; i++) { - var ptr = HEAP32[(((iovs)+(i*8))>>2)]; - var len = HEAP32[(((iovs)+(i*8 + 4))>>2)]; - for (var j = 0; j < len; j++) { - if (HEAPU8[ptr+j] === 0 || HEAPU8[ptr+j] === 10) { - (fd === 1 ? out : err)(UTF8ArrayToString(buffer, 0)); - buffer.length = 0; - } else { - buffer.push(HEAPU8[ptr+j]); - } - } - num += len; - } - HEAP32[((nwritten)>>2)] = num; - }, fd, iovs, iovs_len, nwritten); - } - return 0; - } - - __wasi_errno_t __wasi_fd_seek(__wasi_fd_t fd, __wasi_filedelta_t offset, __wasi_whence_t whence, __wasi_filesize_t *newoffset) - { - abort(); - } - - __wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) - { - abort(); - } - - __wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, size_t iovs_len, __wasi_size_t *nread) - { - abort(); - } -} diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp new file mode 100644 index 0000000000000..00482f7e8f857 --- /dev/null +++ b/system/lib/wasmfs/wasmfs.cpp @@ -0,0 +1,72 @@ +// // Copyright 2021 The Emscripten Authors. All rights reserved. +// // Emscripten is available under two separate licenses, the MIT license and the +// // University of Illinois/NCSA Open Source License. Both these licenses can be +// // found in the LICENSE file. +// // wasmfs.cpp will implement a new file system that replaces the existing JS filesystem +// // Current Status: Work in Progress. +// // See https://github.com/emscripten-core/emscripten/issues/15041 + +#include +#define __NEED_struct_iovec +#include +#include + +extern "C" { + +__wasi_errno_t __wasi_fd_write( + __wasi_fd_t fd, const __wasi_ciovec_t* iovs, size_t iovs_len, __wasi_size_t* nwritten) { + // FD 1 = STDOUT + // FD 2 = STDERR + // Temporary hardcoding of filedescriptor values + // TODO: May not want to proxy stderr (fd == 2) to the main thread + // This will not not show in HTML, a console.warn in a worker is suffficient + // This would be a change from the current FS + if (fd == 1 || fd == 2) { + MAIN_THREAD_EM_ASM( + { + var buffer = []; + var fd = $0; + var iovs = $1; + var iovs_len = $2; + var nwritten = $3; + var num = 0; + for (var i = 0; i < iovs_len; i++) { + var ptr = HEAP32[iovs + i * 8 >> 2]; + var len = HEAP32[iovs + i * 8 + 4 >> 2]; + for (var j = 0; j < len; j++) { + if (HEAPU8[ptr + j] == 0 || HEAPU8[ptr + j] == 10) { + (fd == 1 ? out : err)(UTF8ArrayToString(buffer, 0)); + buffer.length = 0; + } else { + buffer.push(HEAPU8[ptr + j]); + } + } + num += len; + } + HEAP32[((nwritten) >> 2)] = num; + }, + fd, iovs, iovs_len, nwritten); + } + return 0; +} + +__wasi_errno_t __wasi_fd_seek( + __wasi_fd_t fd, __wasi_filedelta_t offset, __wasi_whence_t whence, __wasi_filesize_t* newoffset) { + EM_ASM({out(`${$0} or __wasi_fd_seek has been temporarily stubbed and is inert`)}, + __PRETTY_FUNCTION__); + abort(); +} + +__wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) { + EM_ASM({out(`${$0} or __wasi_fd_close has been temporarily stubbed and is inert`)}, + __PRETTY_FUNCTION__); + abort(); +} + +__wasi_errno_t __wasi_fd_read( + __wasi_fd_t fd, const __wasi_iovec_t* iovs, size_t iovs_len, __wasi_size_t* nread) { + EM_ASM({out(`${$0} or __wasi_fd_read has been temporarily stubbed and is inert`)}, + __PRETTY_FUNCTION__); + abort(); +} +} diff --git a/tests/core/test_wasmfs_hello_world.c b/tests/core/test_wasmfs_hello_world.c new file mode 100644 index 0000000000000..21cc9440d2b63 --- /dev/null +++ b/tests/core/test_wasmfs_hello_world.c @@ -0,0 +1,14 @@ +/* + * Copyright 2021 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +// Testing wasmfs to show library is built and linked successfully + +#include +int main() { + printf("wasmfs!\n"); + return 0; +} diff --git a/tests/core/test_wasmfs_hello_world.out b/tests/core/test_wasmfs_hello_world.out new file mode 100644 index 0000000000000..2258f23197a25 --- /dev/null +++ b/tests/core/test_wasmfs_hello_world.out @@ -0,0 +1 @@ +wasmfs! \ No newline at end of file diff --git a/tests/test_core.py b/tests/test_core.py index d934b1c4dc283..a09cbc107a09a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -139,6 +139,12 @@ def can_do_standalone(self): '-fsanitize=address' not in self.emcc_args +def also_with_wasmfs(func): + def decorated(self): + self.set_setting('WASMFS') + return decorated + + # Impure means a test that cannot run in a wasm VM yet, as it is not 100% # standalone. We can still run them with the JS code though. def also_with_standalone_wasm(wasm2c=False, impure=False): @@ -313,6 +319,11 @@ def test_hello_world(self): # must not emit this unneeded internal thing self.assertNotContained('EMSCRIPTEN_GENERATED_FUNCTIONS', read_file('test_hello_world.js')) + @also_with_wasmfs + def test_wasmfs_hello_world(self): + self.do_core_test('test_wasmfs_hello_world.c') + self.assertNotContained('EMSCRIPTEN_GENERATED_FUNCTIONS', read_file('test_wasmfs_hello_world.js')) + def test_wasm_synchronous_compilation(self): self.set_setting('STRICT_JS') self.set_setting('WASM_ASYNC_COMPILATION', 0) diff --git a/tools/system_libs.py b/tools/system_libs.py index add54a4012247..c1386be6b5234 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1288,7 +1288,7 @@ class libwasmfs(MTLibrary): name = 'libwasmfs' def get_files(self): - return [utils.path_from_root('system/lib/fetch/wasmfs.cpp')] + return [utils.path_from_root('system/lib/wasmfs/wasmfs.cpp')] class libhtml5(Library): From 39729101283ea0d38bdd744908d139f09fa32c7d Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Thu, 23 Sep 2021 11:27:40 -0400 Subject: [PATCH 09/20] white space --- tests/core/test_wasmfs_hello_world.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_wasmfs_hello_world.out b/tests/core/test_wasmfs_hello_world.out index 2258f23197a25..d41c06a9984e3 100644 --- a/tests/core/test_wasmfs_hello_world.out +++ b/tests/core/test_wasmfs_hello_world.out @@ -1 +1 @@ -wasmfs! \ No newline at end of file +wasmfs! From a301fac4e8c40fd3cbcd62cc810a63911c6dc2be Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Thu, 23 Sep 2021 11:59:45 -0400 Subject: [PATCH 10/20] remove parens --- system/lib/wasmfs/wasmfs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 00482f7e8f857..da13a6bb2a506 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -43,7 +43,7 @@ __wasi_errno_t __wasi_fd_write( } num += len; } - HEAP32[((nwritten) >> 2)] = num; + HEAP32[nwritten >> 2] = num; }, fd, iovs, iovs_len, nwritten); } From 117be99148b1c5d17b0a83a1a33b030967769583 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Thu, 23 Sep 2021 16:37:46 -0400 Subject: [PATCH 11/20] grammar and tests fix --- src/settings.js | 6 +++--- system/lib/wasmfs/wasmfs.cpp | 21 ++++++++++----------- tests/common.py | 1 + tests/core/test_wasmfs_hello_world.c | 14 -------------- tests/core/test_wasmfs_hello_world.out | 1 - tests/test_core.py | 10 +++++----- 6 files changed, 19 insertions(+), 34 deletions(-) delete mode 100644 tests/core/test_wasmfs_hello_world.c delete mode 100644 tests/core/test_wasmfs_hello_world.out diff --git a/src/settings.js b/src/settings.js index 2d6b12d94428d..e6d0296d218b4 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1635,9 +1635,9 @@ var FETCH = 0; // [link] var ASMFS = 0; -// ATTENTION [WIP]: Experimental feature, use at your own risk -// Will eventually replace current JS file system implementation -// If set to 1, uses new filesystem implementation +// ATTENTION [WIP]: Experimental feature. Please use at your own risk. +// This will eventually replace the current JS file system implementation. +// If set to 1, uses new filesystem implementation. // [link] var WASMFS = 0; diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index da13a6bb2a506..4ee576af88134 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -2,9 +2,9 @@ // // Emscripten is available under two separate licenses, the MIT license and the // // University of Illinois/NCSA Open Source License. Both these licenses can be // // found in the LICENSE file. -// // wasmfs.cpp will implement a new file system that replaces the existing JS filesystem +// // wasmfs.cpp will implement a new file system that replaces the existing JS filesystem. // // Current Status: Work in Progress. -// // See https://github.com/emscripten-core/emscripten/issues/15041 +// // See https://github.com/emscripten-core/emscripten/issues/15041. #include #define __NEED_struct_iovec @@ -15,12 +15,11 @@ extern "C" { __wasi_errno_t __wasi_fd_write( __wasi_fd_t fd, const __wasi_ciovec_t* iovs, size_t iovs_len, __wasi_size_t* nwritten) { - // FD 1 = STDOUT - // FD 2 = STDERR - // Temporary hardcoding of filedescriptor values - // TODO: May not want to proxy stderr (fd == 2) to the main thread - // This will not not show in HTML, a console.warn in a worker is suffficient - // This would be a change from the current FS + // FD 1 = STDOUT and FD 2 = STDERR. + // Temporary hardcoding of filedescriptor values. + // TODO: May not want to proxy stderr (fd == 2) to the main thread. + // This will not not show in HTML, a console.warn in a worker is suffficient. + // This would be a change from the current FS. if (fd == 1 || fd == 2) { MAIN_THREAD_EM_ASM( { @@ -52,20 +51,20 @@ __wasi_errno_t __wasi_fd_write( __wasi_errno_t __wasi_fd_seek( __wasi_fd_t fd, __wasi_filedelta_t offset, __wasi_whence_t whence, __wasi_filesize_t* newoffset) { - EM_ASM({out(`${$0} or __wasi_fd_seek has been temporarily stubbed and is inert`)}, + EM_ASM({out($0 + ' or __wasi_fd_seek has been temporarily stubbed and is inert')}, __PRETTY_FUNCTION__); abort(); } __wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) { - EM_ASM({out(`${$0} or __wasi_fd_close has been temporarily stubbed and is inert`)}, + EM_ASM({out($0 + ' or __wasi_fd_close has been temporarily stubbed and is inert')}, __PRETTY_FUNCTION__); abort(); } __wasi_errno_t __wasi_fd_read( __wasi_fd_t fd, const __wasi_iovec_t* iovs, size_t iovs_len, __wasi_size_t* nread) { - EM_ASM({out(`${$0} or __wasi_fd_read has been temporarily stubbed and is inert`)}, + EM_ASM({out($0 + ' or __wasi_fd_read has been temporarily stubbed and is inert')}, __PRETTY_FUNCTION__); abort(); } diff --git a/tests/common.py b/tests/common.py index 5f8cf92e1044a..0f1d76cfc241d 100644 --- a/tests/common.py +++ b/tests/common.py @@ -541,6 +541,7 @@ def verify_es5(self, filename): es_check_env = os.environ.copy() es_check_env['PATH'] = os.path.dirname(config.NODE_JS[0]) + os.pathsep + es_check_env['PATH'] try: + # Comment out --quiet if more detailed error logging is needed shared.run_process(es_check + ['es5', os.path.abspath(filename), '--quiet'], stderr=PIPE, env=es_check_env) except subprocess.CalledProcessError as e: print(e.stderr) diff --git a/tests/core/test_wasmfs_hello_world.c b/tests/core/test_wasmfs_hello_world.c deleted file mode 100644 index 21cc9440d2b63..0000000000000 --- a/tests/core/test_wasmfs_hello_world.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2021 The Emscripten Authors. All rights reserved. - * Emscripten is available under two separate licenses, the MIT license and the - * University of Illinois/NCSA Open Source License. Both these licenses can be - * found in the LICENSE file. - */ - -// Testing wasmfs to show library is built and linked successfully - -#include -int main() { - printf("wasmfs!\n"); - return 0; -} diff --git a/tests/core/test_wasmfs_hello_world.out b/tests/core/test_wasmfs_hello_world.out deleted file mode 100644 index d41c06a9984e3..0000000000000 --- a/tests/core/test_wasmfs_hello_world.out +++ /dev/null @@ -1 +0,0 @@ -wasmfs! diff --git a/tests/test_core.py b/tests/test_core.py index a09cbc107a09a..c5e690cfb35c6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -141,7 +141,11 @@ def can_do_standalone(self): def also_with_wasmfs(func): def decorated(self): + self.set_setting('WASMFS', 0) + func(self) + print('wasmfs') self.set_setting('WASMFS') + func(self) return decorated @@ -314,16 +318,12 @@ def get_bullet_library(self, use_cmake): cache_name_extra=configure_commands[0]) @also_with_standalone_wasm() + @also_with_wasmfs def test_hello_world(self): self.do_core_test('test_hello_world.c') # must not emit this unneeded internal thing self.assertNotContained('EMSCRIPTEN_GENERATED_FUNCTIONS', read_file('test_hello_world.js')) - @also_with_wasmfs - def test_wasmfs_hello_world(self): - self.do_core_test('test_wasmfs_hello_world.c') - self.assertNotContained('EMSCRIPTEN_GENERATED_FUNCTIONS', read_file('test_wasmfs_hello_world.js')) - def test_wasm_synchronous_compilation(self): self.set_setting('STRICT_JS') self.set_setting('WASM_ASYNC_COMPILATION', 0) From 9d4bdabd3b943d7883316ad602dfce4ced83c542 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 01:48:12 -0400 Subject: [PATCH 12/20] extracted writing to stdout to library_wasmfs.js --- emcc.py | 2 +- src/library_wasmfs.js | 19 +++++++++++++++++++ src/modules.js | 2 ++ system/lib/wasmfs/wasmfs.cpp | 34 ++++++++++------------------------ 4 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 src/library_wasmfs.js diff --git a/emcc.py b/emcc.py index 9c6e02601b17e..1ed0e4801cc47 100755 --- a/emcc.py +++ b/emcc.py @@ -1771,7 +1771,7 @@ def default_setting(name, new_default): state.forced_stdlibs.append('libwasmfs') settings.FILESYSTEM = 0 settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 - # settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) TODO: Add in future PR + settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. if not settings.USES_DYNAMIC_ALLOC: diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js new file mode 100644 index 0000000000000..e5b4eb301507c --- /dev/null +++ b/src/library_wasmfs.js @@ -0,0 +1,19 @@ +var WasmfsLibrary = { +$WASMFS: { + buffers: [null, [], []], // 1 => stdout, 2 => stderr + printChar: function(stream, curr) { + if (curr === 0 || curr === 10) { + (stream === 1 ? out : err)(UTF8ArrayToString(WASMFS.buffers[stream], 0)); + WASMFS.buffers[stream].length = 0; + } else { + WASMFS.buffers[stream].push(curr); + } + + return + }, +} +} + +mergeInto(LibraryManager.library, WasmfsLibrary); + +DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$WASMFS'); diff --git a/src/modules.js b/src/modules.js index a1167ba9bf44d..1d4433c71b0cf 100644 --- a/src/modules.js +++ b/src/modules.js @@ -110,6 +110,8 @@ var LibraryManager = { } libraries.push('library_noderawfs.js'); } + } else { + libraries.push('library_wasmfs.js'); } // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 4ee576af88134..671dedaea12fe 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -21,30 +21,16 @@ __wasi_errno_t __wasi_fd_write( // This will not not show in HTML, a console.warn in a worker is suffficient. // This would be a change from the current FS. if (fd == 1 || fd == 2) { - MAIN_THREAD_EM_ASM( - { - var buffer = []; - var fd = $0; - var iovs = $1; - var iovs_len = $2; - var nwritten = $3; - var num = 0; - for (var i = 0; i < iovs_len; i++) { - var ptr = HEAP32[iovs + i * 8 >> 2]; - var len = HEAP32[iovs + i * 8 + 4 >> 2]; - for (var j = 0; j < len; j++) { - if (HEAPU8[ptr + j] == 0 || HEAPU8[ptr + j] == 10) { - (fd == 1 ? out : err)(UTF8ArrayToString(buffer, 0)); - buffer.length = 0; - } else { - buffer.push(HEAPU8[ptr + j]); - } - } - num += len; - } - HEAP32[nwritten >> 2] = num; - }, - fd, iovs, iovs_len, nwritten); + __wasi_size_t num = 0; + for (size_t i = 0; i < iovs_len; i++) { + const uint8_t* ptr = iovs[i].buf; + __wasi_size_t len = iovs[i].buf_len; + for (__wasi_size_t j = 0; j < len; j++) { + EM_ASM({ WASMFS.printChar($0, HEAPU8[$1]); }, fd, ptr + j); + } + num += len; + } + *nwritten = num; } return 0; } From 28bcfcbfac07dc849915cb2c345c89028ecd42aa Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 11:47:44 -0400 Subject: [PATCH 13/20] changed layout of library_wasmfs, removed EM_ASM --- src/library_wasmfs.js | 24 +++++++++++------------- system/lib/wasmfs/wasmfs.cpp | 6 +++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index e5b4eb301507c..58552698a044f 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -1,19 +1,17 @@ var WasmfsLibrary = { -$WASMFS: { - buffers: [null, [], []], // 1 => stdout, 2 => stderr - printChar: function(stream, curr) { - if (curr === 0 || curr === 10) { - (stream === 1 ? out : err)(UTF8ArrayToString(WASMFS.buffers[stream], 0)); - WASMFS.buffers[stream].length = 0; - } else { - WASMFS.buffers[stream].push(curr); + $wasmfsBuffers: [null, [], []], + emscripten_wasmfs_printchar__deps: ['$wasmfsBuffers'], + emscripten_wasmfs_printchar: function(stream, ptr, len) { + for (var j = 0; j < len; j++) { + if (HEAPU8[ptr+j] === 0 || HEAPU8[ptr+j] === 10) { + (stream === 1 ? out : err)(UTF8ArrayToString(wasmfsBuffers[stream], 0)); + wasmfsBuffers[stream].length = 0; + } else { + wasmfsBuffers[stream].push(HEAPU8[ptr+j]); + } } - return - }, -} + } } mergeInto(LibraryManager.library, WasmfsLibrary); - -DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$WASMFS'); diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 671dedaea12fe..66e2c4ae958bb 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -13,6 +13,8 @@ extern "C" { +int emscripten_wasmfs_printchar(__wasi_fd_t fd, const uint8_t* ptr, __wasi_size_t len); + __wasi_errno_t __wasi_fd_write( __wasi_fd_t fd, const __wasi_ciovec_t* iovs, size_t iovs_len, __wasi_size_t* nwritten) { // FD 1 = STDOUT and FD 2 = STDERR. @@ -25,9 +27,7 @@ __wasi_errno_t __wasi_fd_write( for (size_t i = 0; i < iovs_len; i++) { const uint8_t* ptr = iovs[i].buf; __wasi_size_t len = iovs[i].buf_len; - for (__wasi_size_t j = 0; j < len; j++) { - EM_ASM({ WASMFS.printChar($0, HEAPU8[$1]); }, fd, ptr + j); - } + emscripten_wasmfs_printchar(fd, ptr, len); num += len; } *nwritten = num; From d31597ff03723ad1686d6e0322d107ac78358784 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 12:59:04 -0400 Subject: [PATCH 14/20] change name to _printbuffer --- src/library_wasmfs.js | 4 ++-- system/lib/wasmfs/wasmfs.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index 58552698a044f..0ae01069a8a47 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -1,7 +1,7 @@ var WasmfsLibrary = { $wasmfsBuffers: [null, [], []], - emscripten_wasmfs_printchar__deps: ['$wasmfsBuffers'], - emscripten_wasmfs_printchar: function(stream, ptr, len) { + emscripten_wasmfs_printbuffer__deps: ['$wasmfsBuffers'], + emscripten_wasmfs_printbuffer: function(stream, ptr, len) { for (var j = 0; j < len; j++) { if (HEAPU8[ptr+j] === 0 || HEAPU8[ptr+j] === 10) { (stream === 1 ? out : err)(UTF8ArrayToString(wasmfsBuffers[stream], 0)); diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 66e2c4ae958bb..cc3dd5f6960ec 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -13,7 +13,7 @@ extern "C" { -int emscripten_wasmfs_printchar(__wasi_fd_t fd, const uint8_t* ptr, __wasi_size_t len); +int emscripten_wasmfs_printbuffer(__wasi_fd_t fd, const uint8_t* ptr, __wasi_size_t len); __wasi_errno_t __wasi_fd_write( __wasi_fd_t fd, const __wasi_ciovec_t* iovs, size_t iovs_len, __wasi_size_t* nwritten) { @@ -27,7 +27,7 @@ __wasi_errno_t __wasi_fd_write( for (size_t i = 0; i < iovs_len; i++) { const uint8_t* ptr = iovs[i].buf; __wasi_size_t len = iovs[i].buf_len; - emscripten_wasmfs_printchar(fd, ptr, len); + emscripten_wasmfs_printbuffer(fd, ptr, len); num += len; } *nwritten = num; From 0ee2972b1ee0654bd8ee91d40243d8b752f8fc42 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 14:06:10 -0400 Subject: [PATCH 15/20] use emscripten_console_log --- system/lib/wasmfs/wasmfs.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index cc3dd5f6960ec..88a5cb21f8c20 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -37,21 +37,18 @@ __wasi_errno_t __wasi_fd_write( __wasi_errno_t __wasi_fd_seek( __wasi_fd_t fd, __wasi_filedelta_t offset, __wasi_whence_t whence, __wasi_filesize_t* newoffset) { - EM_ASM({out($0 + ' or __wasi_fd_seek has been temporarily stubbed and is inert')}, - __PRETTY_FUNCTION__); + emscripten_console_log("__wasi_fd_seek has been temporarily stubbed and is inert"); abort(); } __wasi_errno_t __wasi_fd_close(__wasi_fd_t fd) { - EM_ASM({out($0 + ' or __wasi_fd_close has been temporarily stubbed and is inert')}, - __PRETTY_FUNCTION__); + emscripten_console_log("__wasi_fd_close has been temporarily stubbed and is inert"); abort(); } __wasi_errno_t __wasi_fd_read( __wasi_fd_t fd, const __wasi_iovec_t* iovs, size_t iovs_len, __wasi_size_t* nread) { - EM_ASM({out($0 + ' or __wasi_fd_read has been temporarily stubbed and is inert')}, - __PRETTY_FUNCTION__); + emscripten_console_log("__wasi_fd_read has been temporarily stubbed and is inert"); abort(); } } From 47d4392c67f5458296e328396c107411837d75de Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 14:25:13 -0400 Subject: [PATCH 16/20] add html5.h header --- system/lib/wasmfs/wasmfs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 88a5cb21f8c20..9610d5704d1d1 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -9,6 +9,7 @@ #include #define __NEED_struct_iovec #include +#include #include extern "C" { From 13ef89f08c5038cf1021485bf304f479329b2c23 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 16:25:44 -0400 Subject: [PATCH 17/20] removed errant #define, update test --- system/lib/wasmfs/wasmfs.cpp | 3 +-- tests/test_core.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 9610d5704d1d1..5dfa7fb3fe167 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -6,10 +6,9 @@ // // Current Status: Work in Progress. // // See https://github.com/emscripten-core/emscripten/issues/15041. -#include -#define __NEED_struct_iovec #include #include +#include #include extern "C" { diff --git a/tests/test_core.py b/tests/test_core.py index 14a09cf456d6f..670f0f4901ad4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -144,6 +144,8 @@ def decorated(self): self.set_setting('WASMFS', 0) func(self) print('wasmfs') + if self.get_setting('STANDALONE_WASM'): + self.skipTest("test currently cannot run both with WASMFS and STANDALONE_WASM") self.set_setting('WASMFS') func(self) return decorated From 8ce49c41a899773fa485f937d93acf02eedfe915 Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 17:36:50 -0400 Subject: [PATCH 18/20] edited tests, comments --- system/lib/wasmfs/wasmfs.cpp | 14 +++++++------- tests/test_core.py | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 5dfa7fb3fe167..dc4e8e75e1bee 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -1,10 +1,10 @@ -// // Copyright 2021 The Emscripten Authors. All rights reserved. -// // Emscripten is available under two separate licenses, the MIT license and the -// // University of Illinois/NCSA Open Source License. Both these licenses can be -// // found in the LICENSE file. -// // wasmfs.cpp will implement a new file system that replaces the existing JS filesystem. -// // Current Status: Work in Progress. -// // See https://github.com/emscripten-core/emscripten/issues/15041. +// Copyright 2021 The Emscripten Authors. All rights reserved. +// Emscripten is available under two separate licenses, the MIT license and the +// University of Illinois/NCSA Open Source License. Both these licenses can be +// found in the LICENSE file. +// wasmfs.cpp will implement a new file system that replaces the existing JS filesystem. +// Current Status: Work in Progress. +// See https://github.com/emscripten-core/emscripten/issues/15041. #include #include diff --git a/tests/test_core.py b/tests/test_core.py index 670f0f4901ad4..d99e02c18f1e4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -141,7 +141,6 @@ def can_do_standalone(self): def also_with_wasmfs(func): def decorated(self): - self.set_setting('WASMFS', 0) func(self) print('wasmfs') if self.get_setting('STANDALONE_WASM'): From d86d65590098bc2939659dc211fe5c278496958a Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 18:12:55 -0400 Subject: [PATCH 19/20] if WASMFS is off, do not link wasmfs.cpp --- tools/system_libs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/system_libs.py b/tools/system_libs.py index 3ce81fa7ca9a0..58175bf22db1a 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1290,6 +1290,9 @@ class libwasmfs(MTLibrary): def get_files(self): return [utils.path_from_root('system/lib/wasmfs/wasmfs.cpp')] + + def can_build(self): + return settings.WASMFS class libhtml5(Library): From d9936bf8aa443cded6b4895ddb5e7f893f76c9fe Mon Sep 17 00:00:00 2001 From: Ethan Lee Date: Fri, 24 Sep 2021 18:43:26 -0400 Subject: [PATCH 20/20] remove whitespace --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 58175bf22db1a..e354283f881a1 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1290,7 +1290,7 @@ class libwasmfs(MTLibrary): def get_files(self): return [utils.path_from_root('system/lib/wasmfs/wasmfs.cpp')] - + def can_build(self): return settings.WASMFS