From c606386ca53b8af2d66b1b2afc106bfc8ea663dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Thu, 1 May 2025 18:12:57 +0200 Subject: [PATCH 01/20] Implement `open`/`stat`/`mkdir`/`symlink` for standalone WASI mode --- system/lib/standalone/paths.c | 228 +++++++++++++++++++++++++++++ system/lib/standalone/paths.h | 21 +++ system/lib/standalone/standalone.c | 199 +++++++++++++++++++++++-- test/common.py | 48 ++++-- test/jsrun.py | 7 +- test/test_core.py | 1 + tools/system_libs.py | 1 + 7 files changed, 480 insertions(+), 25 deletions(-) create mode 100644 system/lib/standalone/paths.c create mode 100644 system/lib/standalone/paths.h diff --git a/system/lib/standalone/paths.c b/system/lib/standalone/paths.c new file mode 100644 index 0000000000000..84e671287abe4 --- /dev/null +++ b/system/lib/standalone/paths.c @@ -0,0 +1,228 @@ +#define _GNU_SOURCE +#include "paths.h" + +#include +#include +#include +#include +#include +#include +#include + +/// A name and file descriptor pair. +typedef struct preopen { + /// The path prefix associated with the file descriptor. + const char* prefix; + + /// The file descriptor. + __wasi_fd_t fd; +} preopen; + +/// A simple growable array of `preopen`. +static preopen* preopens; +static size_t num_preopens; +static size_t preopen_capacity; + +#ifdef NDEBUG +#define assert_invariants() // assertions disabled +#else +static void assert_invariants(void) { + assert(num_preopens <= preopen_capacity); + assert(preopen_capacity == 0 || preopens != NULL); + assert(preopen_capacity == 0 || + preopen_capacity * sizeof(preopen) > preopen_capacity); + + for (size_t i = 0; i < num_preopens; ++i) { + const preopen* pre = &preopens[i]; + assert(pre->prefix != NULL); + assert(pre->fd != (__wasi_fd_t)-1); +#ifdef __wasm__ + assert((uintptr_t)pre->prefix < + (__uint128_t)__builtin_wasm_memory_size(0) * PAGESIZE); +#endif + } +} +#endif + +/// Allocate space for more preopens. Returns 0 on success and -1 on failure. +static bool resize_preopens(void) { + size_t start_capacity = 4; + size_t old_capacity = preopen_capacity; + size_t new_capacity = old_capacity == 0 ? start_capacity : old_capacity * 2; + + preopen* old_preopens = preopens; + preopen* new_preopens = calloc(sizeof(preopen), new_capacity); + if (new_preopens == NULL) { + return false; + } + + memcpy(new_preopens, old_preopens, num_preopens * sizeof(preopen)); + preopens = new_preopens; + preopen_capacity = new_capacity; + free(old_preopens); + + assert_invariants(); + return true; +} + +// Normalize an absolute path. Removes leading `/` and leading `./`, so the +// first character is the start of a directory name. This works because our +// process always starts with a working directory of `/`. Additionally translate +// `.` to the empty string. +static const char* strip_prefixes(const char* path) { + while (1) { + if (path[0] == '/') { + path++; + } else if (path[0] == '.' && path[1] == '/') { + path += 2; + } else if (path[0] == '.' && path[1] == 0) { + path++; + } else { + break; + } + } + + return path; +} + +/// Register the given preopened file descriptor under the given path. +/// +/// This function takes ownership of `prefix`. +static bool register_preopened_fd(__wasi_fd_t fd, const char* relprefix) { + // Check preconditions. + assert_invariants(); + assert(fd != AT_FDCWD); + assert(fd != -1); + assert(relprefix != NULL); + + if (num_preopens == preopen_capacity && !resize_preopens()) { + return false; + } + + char* prefix = strdup(strip_prefixes(relprefix)); + if (prefix == NULL) { + return false; + } + preopens[num_preopens++] = (preopen){ + prefix, + fd, + }; + + assert_invariants(); + return true; +} + +/// Are the `prefix_len` bytes pointed to by `prefix` a prefix of `path`? +static bool +prefix_matches(const char* prefix, size_t prefix_len, const char* path) { + // Allow an empty string as a prefix of any relative path. + if (path[0] != '/' && prefix_len == 0) + return true; + + // Check whether any bytes of the prefix differ. + if (memcmp(path, prefix, prefix_len) != 0) + return false; + + // Ignore trailing slashes in directory names. + size_t i = prefix_len; + while (i > 0 && prefix[i - 1] == '/') { + --i; + } + + // Match only complete path components. + char last = path[i]; + return last == '/' || last == '\0'; +} + +bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr) { + const char* path = *path_ptr; + + if (*resolved_dirfd != AT_FDCWD && path[0] != '/') { + return true; + } + + // Strip leading `/` characters, the prefixes we're mataching won't have + // them. + while (*path == '/') + path++; + // Search through the preopens table. Iterate in reverse so that more + // recently added preopens take precedence over less recently addded ones. + size_t match_len = 0; + int fd = -1; + for (size_t i = num_preopens; i > 0; --i) { + const preopen* pre = &preopens[i - 1]; + const char* prefix = pre->prefix; + size_t len = strlen(prefix); + + // If we haven't had a match yet, or the candidate path is longer than + // our current best match's path, and the candidate path is a prefix of + // the requested path, take that as the new best path. + if ((fd == -1 || len > match_len) && prefix_matches(prefix, len, path)) { + fd = pre->fd; + match_len = len; + } + } + + if (fd == -1) { + return false; + } + + // The relative path is the substring after the portion that was matched. + const char* computed = path + match_len; + + // Omit leading slashes in the relative path. + while (*computed == '/') + ++computed; + + // *at syscalls don't accept empty relative paths, so use "." instead. + if (*computed == '\0') + computed = "."; + + *resolved_dirfd = fd; + *path_ptr = computed; + return true; +} + +// Populate WASI preopens. +__attribute__((constructor(100))) // construct this before user code +static void _standalone_populate_preopens(void) { + // Skip stdin, stdout, and stderr, and count up until we reach an invalid + // file descriptor. + for (__wasi_fd_t fd = 3; fd != 0; ++fd) { + __wasi_prestat_t prestat; + __wasi_errno_t ret = __wasi_fd_prestat_get(fd, &prestat); + if (ret == __WASI_ERRNO_BADF) + break; + if (ret != __WASI_ERRNO_SUCCESS) + goto oserr; + switch (prestat.pr_type) { + case __WASI_PREOPENTYPE_DIR: { + char* prefix = malloc(prestat.u.dir.pr_name_len + 1); + if (prefix == NULL) + goto software; + + // TODO: Remove the cast on `path` once the witx is updated with + // char8 support. + ret = __wasi_fd_prestat_dir_name( + fd, (uint8_t*)prefix, prestat.u.dir.pr_name_len); + if (ret != __WASI_ERRNO_SUCCESS) + goto oserr; + prefix[prestat.u.dir.pr_name_len] = '\0'; + + if (!register_preopened_fd(fd, prefix)) + goto software; + free(prefix); + + break; + } + default: + break; + } + } + + return; +oserr: + _Exit(EX_OSERR); +software: + _Exit(EX_SOFTWARE); +} diff --git a/system/lib/standalone/paths.h b/system/lib/standalone/paths.h new file mode 100644 index 0000000000000..ceafd9da3b0e6 --- /dev/null +++ b/system/lib/standalone/paths.h @@ -0,0 +1,21 @@ +#ifndef STANDALONE_PATHS_H +#define STANDALONE_PATHS_H + +#include + +// +// Resolve a (dirfd, relative/absolute path) pair. +// +// Arguments: +// - `resolved_dirfd`: +// - as input: input dirfd, may be `AT_FDCWD` +// - as output: resolved dirfd (which always is a preopened fd) +// - `path_ptr`: +// - as input: pointer to a relative or absolute path +// - as output: a path relative to `resolved_dirfd` +// +// Returns: `true` if resolution was successful, `false` otherwise. +// +bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr); + +#endif diff --git a/system/lib/standalone/standalone.c b/system/lib/standalone/standalone.c index 3a910ba662d62..e5b2a685c6827 100644 --- a/system/lib/standalone/standalone.c +++ b/system/lib/standalone/standalone.c @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -25,6 +27,7 @@ #include "lock.h" #include "emscripten_internal.h" +#include "paths.h" /* * WASI support code. These are compiled with the program, and call out @@ -45,6 +48,42 @@ _Static_assert(CLOCK_MONOTONIC == __WASI_CLOCKID_MONOTONIC, "must match"); _Static_assert(CLOCK_PROCESS_CPUTIME_ID == __WASI_CLOCKID_PROCESS_CPUTIME_ID, "must match"); _Static_assert(CLOCK_THREAD_CPUTIME_ID == __WASI_CLOCKID_THREAD_CPUTIME_ID, "must match"); +static void wasi_filestat_to_stat(const __wasi_filestat_t* in, + struct stat* out) { + *out = (struct stat){ + .st_dev = in->dev, + .st_ino = in->ino, + .st_nlink = in->nlink, + .st_size = in->size, + .st_atim = __wasi_timestamp_to_timespec(in->atim), + .st_mtim = __wasi_timestamp_to_timespec(in->mtim), + .st_ctim = __wasi_timestamp_to_timespec(in->ctim), + }; + + // Convert file type to legacy types encoded in st_mode. + switch (in->filetype) { + case __WASI_FILETYPE_BLOCK_DEVICE: + out->st_mode |= S_IFBLK; + break; + case __WASI_FILETYPE_CHARACTER_DEVICE: + out->st_mode |= S_IFCHR; + break; + case __WASI_FILETYPE_DIRECTORY: + out->st_mode |= S_IFDIR; + break; + case __WASI_FILETYPE_REGULAR_FILE: + out->st_mode |= S_IFREG; + break; + case __WASI_FILETYPE_SOCKET_DGRAM: + case __WASI_FILETYPE_SOCKET_STREAM: + out->st_mode |= S_IFSOCK; + break; + case __WASI_FILETYPE_SYMBOLIC_LINK: + out->st_mode |= S_IFLNK; + break; + } +} + // mmap support is nonexistent. TODO: emulate simple mmaps using // stdio + malloc, which is slow but may help some things? @@ -65,21 +104,106 @@ weak int _munmap_js( return -ENOSYS; } -// open(), etc. - we just support the standard streams, with no -// corner case error checking; everything else is not permitted. -// TODO: full file support for WASI, or an option for it -// open() weak int __syscall_openat(int dirfd, intptr_t path, int flags, ...) { - if (!strcmp((const char*)path, "/dev/stdin")) { + const char* resolved_path = (const char*)path; + + if (!strcmp(resolved_path, "/dev/stdin")) { return STDIN_FILENO; } - if (!strcmp((const char*)path, "/dev/stdout")) { + if (!strcmp(resolved_path, "/dev/stdout")) { return STDOUT_FILENO; } - if (!strcmp((const char*)path, "/dev/stderr")) { + if (!strcmp(resolved_path, "/dev/stderr")) { return STDERR_FILENO; } - return -EPERM; + + if (!__paths_resolve_path(&dirfd, &resolved_path)) { + return -ENOENT; + } + + // Compute rights corresponding with the access modes provided. + // Attempt to obtain all rights, except the ones that contradict the + // access mode provided to openat(). + __wasi_rights_t max = + ~(__WASI_RIGHTS_FD_DATASYNC | __WASI_RIGHTS_FD_READ | + __WASI_RIGHTS_FD_WRITE | __WASI_RIGHTS_FD_ALLOCATE | + __WASI_RIGHTS_FD_READDIR | __WASI_RIGHTS_FD_FILESTAT_SET_SIZE); + { + int accmode = flags & O_ACCMODE; + if (accmode == O_RDONLY || accmode == O_RDWR || accmode == O_WRONLY) { + if (accmode == O_RDONLY || accmode == O_RDWR) { + max |= __WASI_RIGHTS_FD_READ | __WASI_RIGHTS_FD_READDIR; + } + if (accmode == O_WRONLY || accmode == O_RDWR) { + max |= __WASI_RIGHTS_FD_DATASYNC | __WASI_RIGHTS_FD_WRITE | + __WASI_RIGHTS_FD_ALLOCATE | + __WASI_RIGHTS_FD_FILESTAT_SET_SIZE; + } + } else if (accmode == O_EXEC || accmode == O_SEARCH) { + // Do nothing. + } else { + return -EINVAL; + } + } + + // Ensure that we can actually obtain the minimal rights needed. + __wasi_fdstat_t fsb_cur; + __wasi_errno_t error = __wasi_fd_fdstat_get(dirfd, &fsb_cur); + if (error != __WASI_ERRNO_SUCCESS) { + return -error; + } + + // Path lookup properties. + __wasi_lookupflags_t lookup_flags = 0; + if ((flags & O_NOFOLLOW) == 0) { + lookup_flags |= __WASI_LOOKUPFLAGS_SYMLINK_FOLLOW; + } + + // Open file with appropriate rights. + __wasi_fdflags_t fs_flags = 0; + if (flags & O_APPEND) { + fs_flags |= __WASI_FDFLAGS_APPEND; + } + if (flags & O_DSYNC) { + fs_flags |= __WASI_FDFLAGS_DSYNC; + } + if (flags & O_NONBLOCK) { + fs_flags |= __WASI_FDFLAGS_NONBLOCK; + } + if (flags & O_RSYNC) { + fs_flags |= __WASI_FDFLAGS_RSYNC; + } + if (flags & O_SYNC) { + fs_flags |= __WASI_FDFLAGS_SYNC; + } + + __wasi_oflags_t oflags = 0; + if (flags & O_CREAT) { + oflags |= __WASI_OFLAGS_CREAT; + } + if (flags & O_DIRECTORY) { + oflags |= __WASI_OFLAGS_DIRECTORY; + } + if (flags & O_EXCL) { + oflags |= __WASI_OFLAGS_EXCL; + } + if (flags & O_TRUNC) { + oflags |= __WASI_OFLAGS_TRUNC; + } + + __wasi_rights_t fs_rights_base = max & fsb_cur.fs_rights_inheriting; + __wasi_rights_t fs_rights_inheriting = fsb_cur.fs_rights_inheriting; + __wasi_fd_t newfd; + + error = __wasi_path_open(dirfd, lookup_flags, resolved_path, strlen(resolved_path), + oflags, + fs_rights_base, fs_rights_inheriting, fs_flags, + &newfd); + if (error != __WASI_ERRNO_SUCCESS) { + return -error; + } + + return newfd; } weak int __syscall_ioctl(int fd, int op, ...) { @@ -95,7 +219,7 @@ weak int __syscall_fstat64(int fd, intptr_t buf) { } weak int __syscall_stat64(intptr_t path, intptr_t buf) { - return -ENOSYS; + return __syscall_newfstatat(AT_FDCWD, path, buf, 0); } weak int __syscall_dup(int fd) { @@ -103,15 +227,66 @@ weak int __syscall_dup(int fd) { } weak int __syscall_mkdirat(int dirfd, intptr_t path, int mode) { - return -ENOSYS; + const char* resolved_path = (const char*)path; + + if (!__paths_resolve_path(&dirfd, &resolved_path)) { + return -ENOENT; + } + + __wasi_errno_t error = __wasi_path_create_directory(dirfd, resolved_path, strlen(resolved_path)); + if (error != __WASI_ERRNO_SUCCESS) { + return -error; + } + return 0; } weak int __syscall_newfstatat(int dirfd, intptr_t path, intptr_t buf, int flags) { - return -ENOSYS; + // Convert flags to WASI. + __wasi_lookupflags_t lookup_flags = 0; + if ((flags & AT_SYMLINK_NOFOLLOW) == 0) { + lookup_flags |= __WASI_LOOKUPFLAGS_SYMLINK_FOLLOW; + } + + const char* resolved_path = (const char*)path; + + if (!__paths_resolve_path(&dirfd, &resolved_path)) { + return -ENOENT; + } + + __wasi_filestat_t fsb_cur; + __wasi_errno_t error = __wasi_path_filestat_get( + dirfd, lookup_flags, resolved_path, strlen(resolved_path), &fsb_cur); + if (error != __WASI_ERRNO_SUCCESS) { + return -error; + } + + wasi_filestat_to_stat(&fsb_cur, (struct stat*)buf); + + return 0; } weak int __syscall_lstat64(intptr_t path, intptr_t buf) { - return -ENOSYS; + return __syscall_newfstatat(AT_FDCWD, path, buf, AT_SYMLINK_NOFOLLOW); +} + +weak int __syscall_symlinkat(intptr_t target_arg, int newdirfd, intptr_t linkpath) { + const char* resolved_linkpath = (const char*)linkpath; + + if (!__paths_resolve_path(&newdirfd, &resolved_linkpath)) { + return -ENOENT; + } + + const char* target = (const char*)target_arg; + + __wasi_errno_t error = __wasi_path_symlink(target, + strlen(target), + newdirfd, + resolved_linkpath, + strlen(resolved_linkpath)); + if (error != __WASI_ERRNO_SUCCESS) { + return -error; + } + return 0; } // Emscripten additions diff --git a/test/common.py b/test/common.py index beb266d2d0021..4863a9b5bfa22 100644 --- a/test/common.py +++ b/test/common.py @@ -614,7 +614,7 @@ def can_do_standalone(self, impure=False): # Impure means a test that cannot run in a wasm VM yet, as it is not 100% # standalone. We can still run them with the JS code though. -def also_with_standalone_wasm(impure=False): +def also_with_standalone_wasm(impure=False, exclude_engines=None): def decorated(func): @wraps(func) def metafunc(self, standalone): @@ -623,11 +623,19 @@ def metafunc(self, standalone): if not standalone: func(self) else: + nonlocal exclude_engines + if exclude_engines is None: + exclude_engines = [] if not can_do_standalone(self, impure): self.skipTest('Test configuration is not compatible with STANDALONE_WASM') self.set_setting('STANDALONE_WASM') if not impure: self.set_setting('PURE_WASI') + if 'node' in exclude_engines: + # When not running under node we don't care for any undefined symbols + # in the .js as we are only interested in the .wasm file. + self.set_setting('ERROR_ON_UNDEFINED_SYMBOLS=0') + self.emcc_args.append('-Wno-js-compiler') # we will not legalize the JS ffi interface, so we must use BigInt # support in order for JS to have a chance to run this without trapping # when it sees an i64 on the ffi. @@ -636,8 +644,14 @@ def metafunc(self, standalone): # if we are impure, disallow all wasm engines if impure: self.wasm_engines = [] - nodejs = self.require_node() - self.node_args += shared.node_bigint_flags(nodejs) + else: + self.wasm_engines = [engine for engine in self.wasm_engines + if all([not excluded in os.path.basename(engine[0]) for excluded in exclude_engines])] + if 'node' in exclude_engines: + self.js_engines = [] + else: + nodejs = self.require_node(allow_wasm_engines=True) + self.node_args += shared.node_bigint_flags(nodejs) func(self) parameterize(metafunc, {'': (False,), @@ -981,14 +995,14 @@ def get_nodejs(self): return None return config.NODE_JS_TEST - def require_node(self): + def require_node(self, allow_wasm_engines=False): nodejs = self.get_nodejs() if not nodejs: if 'EMTEST_SKIP_NODE' in os.environ: self.skipTest('test requires node and EMTEST_SKIP_NODE is set') else: self.fail('node required to run this test. Use EMTEST_SKIP_NODE to skip') - self.require_engine(nodejs) + self.require_engine(nodejs, allow_wasm_engines) return nodejs def node_is_canary(self, nodejs): @@ -1005,13 +1019,14 @@ def require_node_canary(self): else: self.fail('node canary required to run this test. Use EMTEST_SKIP_NODE_CANARY to skip') - def require_engine(self, engine): + def require_engine(self, engine, allow_wasm_engines=False): logger.debug(f'require_engine: {engine}') if self.required_engine and self.required_engine != engine: self.skipTest(f'Skipping test that requires `{engine}` when `{self.required_engine}` was previously required') self.required_engine = engine self.js_engines = [engine] - self.wasm_engines = [] + if not allow_wasm_engines: + self.wasm_engines = [] def require_wasm64(self): if self.is_browser_test(): @@ -1505,7 +1520,11 @@ def cleanup(line): def run_js(self, filename, engine=None, args=None, assert_returncode=0, interleaved_output=True, - input=None): + input=None, + run_in_tmpdir=False): + if run_in_tmpdir: + ensure_dir(self.in_dir('fs')) + # use files, as PIPE can get too full and hang us stdout_file = self.in_dir('stdout') stderr_file = None @@ -1527,6 +1546,7 @@ def run_js(self, filename, engine=None, args=None, engine = engine + self.spidermonkey_args try: jsrun.run_js(filename, engine, args, + cwd=self.in_dir('fs') if run_in_tmpdir else None, stdout=stdout, stderr=stderr, assert_returncode=assert_returncode, @@ -1562,6 +1582,9 @@ def run_js(self, filename, engine=None, args=None, else: self.fail('JS subprocess failed (%s): %s (expected=%s). Output:\n%s' % (error.cmd, error.returncode, assert_returncode, ret)) + if run_in_tmpdir: + force_delete_contents(self.in_dir('fs')) + return ret def assertExists(self, filename, msg=None): @@ -1957,7 +1980,7 @@ def _build_and_run(self, filename, expected_output, args=None, js_file = self.build(filename, **kwargs) self.assertExists(js_file) - engines = self.js_engines.copy() + engines = [(el, False) for el in self.js_engines.copy()] if len(engines) > 1 and not self.use_all_engines: engines = engines[:1] # In standalone mode, also add wasm vms as we should be able to run there too. @@ -1966,13 +1989,14 @@ def _build_and_run(self, filename, expected_output, args=None, # like with js engines, but for now as we bring it up, test in all of them if not self.wasm_engines: logger.warning('no wasm engine was found to run the standalone part of this test') - engines += self.wasm_engines + engines += [(el, True) for el in self.wasm_engines] if len(engines) == 0: self.fail('No JS engine present to run this test with. Check %s and the paths therein.' % config.EM_CONFIG) - for engine in engines: + for engine, run_in_tmpdir in engines: js_output = self.run_js(js_file, engine, args, assert_returncode=assert_returncode, - interleaved_output=interleaved_output) + interleaved_output=interleaved_output, + run_in_tmpdir=run_in_tmpdir) js_output = js_output.replace('\r\n', '\n') if expected_output: if type(expected_output) not in [list, tuple]: diff --git a/test/jsrun.py b/test/jsrun.py index d7a9cf817922e..e08d7e2219038 100644 --- a/test/jsrun.py +++ b/test/jsrun.py @@ -40,10 +40,15 @@ def make_command(filename, engine, args=None): is_jsc = 'jsc' in jsengine or 'javascriptcore' in jsengine is_wasmer = 'wasmer' in jsengine is_wasmtime = 'wasmtime' in jsengine + is_toywasm = 'toywasm' in jsengine command_flags = [] if is_wasmer: command_flags += ['run'] - if is_wasmer or is_wasmtime: + elif is_wasmtime: + command_flags += ['--dir', '.', '--'] + elif is_toywasm: + command_flags += ['--wasi', '--wasi-dir', '.', '--'] + if is_wasmer or is_wasmtime or is_toywasm: # in a wasm runtime, run the wasm, not the js filename = shared.replace_suffix(filename, '.wasm') # Separates engine flags from script flags diff --git a/test/test_core.py b/test/test_core.py index fe4a0e0159cdf..6df18fc86bb5e 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -5582,6 +5582,7 @@ def test_fcntl(self): @crossplatform @also_with_nodefs_both + @also_with_standalone_wasm(exclude_engines=['node', 'wasmer']) def test_fcntl_open(self): nodefs = '-DNODEFS' in self.emcc_args or '-DNODERAWFS' in self.emcc_args if nodefs and WINDOWS: diff --git a/tools/system_libs.py b/tools/system_libs.py index d82ab077bb3a2..6faaccf01de92 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2217,6 +2217,7 @@ def get_files(self): path='system/lib/standalone', filenames=['standalone.c', 'standalone_wasm_stdio.c', + 'paths.c', '__main_void.c']) # It is more efficient to use JS methods for time, normally. files += files_in_path( From 46b7fa4c2e355382f615dc74abd1f93cd0ac0b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 2 May 2025 13:56:46 +0200 Subject: [PATCH 02/20] remove unneeded list comprehension --- test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index 4863a9b5bfa22..f17a8e451c019 100644 --- a/test/common.py +++ b/test/common.py @@ -646,7 +646,7 @@ def metafunc(self, standalone): self.wasm_engines = [] else: self.wasm_engines = [engine for engine in self.wasm_engines - if all([not excluded in os.path.basename(engine[0]) for excluded in exclude_engines])] + if all(excluded not in os.path.basename(engine[0]) for excluded in exclude_engines)] if 'node' in exclude_engines: self.js_engines = [] else: From 16b1925804880d6503fea4a2e8266213ff85de6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 2 May 2025 14:26:40 +0200 Subject: [PATCH 03/20] skip standalone WASM tests when no engines are configured --- test/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/common.py b/test/common.py index f17a8e451c019..238a9cec6c8f4 100644 --- a/test/common.py +++ b/test/common.py @@ -649,6 +649,8 @@ def metafunc(self, standalone): if all(excluded not in os.path.basename(engine[0]) for excluded in exclude_engines)] if 'node' in exclude_engines: self.js_engines = [] + if not self.wasm_engines: + self.skipTest('no WASM engines available for this test') else: nodejs = self.require_node(allow_wasm_engines=True) self.node_args += shared.node_bigint_flags(nodejs) From e12df0f658707d844c02691bf1892558b35343fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 2 May 2025 15:12:29 +0200 Subject: [PATCH 04/20] update wasmtime version in CI --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2afe0c6b1372..f277a5ecfdcd2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -493,7 +493,7 @@ jobs: name: get wasmtime command: | # use a pinned version due to https://github.com/bytecodealliance/wasmtime/issues/714 - export VERSION=v0.33.0 + export VERSION=v32.0.0 wget https://github.com/bytecodealliance/wasmtime/releases/download/$VERSION/wasmtime-$VERSION-x86_64-linux.tar.xz tar -xf wasmtime-$VERSION-x86_64-linux.tar.xz cp wasmtime-$VERSION-x86_64-linux/wasmtime ~/vms From 65bec8875127bc67494144fbc593e0a33121709a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 2 May 2025 17:17:25 +0200 Subject: [PATCH 05/20] try to fix 'test_time' and 'test_console_out' tests --- test/test_other.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_other.py b/test/test_other.py index 2e0b3ad792370..cb56a02ffee9b 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -6446,7 +6446,8 @@ def test_force_stdlibs(self): # See https://github.com/emscripten-core/emscripten/issues/22161 self.do_runf('hello_world.c', emcc_args=['-sWASM_BIGINT']) - @also_with_standalone_wasm() + # `wasmtime` hangs + @also_with_standalone_wasm(exclude_engines=['wasmtime']) def test_time(self): self.do_other_test('test_time.c') @@ -15401,7 +15402,8 @@ def test_standalone_whole_archive(self): def test_proxy_to_worker(self, args): self.do_runf('hello_world.c', emcc_args=['--proxy-to-worker'] + args) - @also_with_standalone_wasm() + # functions from `emscripten/console.h` only work with node + @also_with_standalone_wasm(impure=True) def test_console_out(self): self.do_other_test('test_console_out.c', regex=True) From ca3f7952122c2aaccc82e61cb7d42af2b6ee306b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 16:19:01 +0200 Subject: [PATCH 06/20] add stubbed out path_filestat_get to libwasi.js --- src/lib/libwasi.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/libwasi.js b/src/lib/libwasi.js index de874ccf868ca..93783cd3b239f 100644 --- a/src/lib/libwasi.js +++ b/src/lib/libwasi.js @@ -612,6 +612,11 @@ var WasiLibrary = { randomFill(HEAPU8.subarray(buffer, buffer + size)); return 0; }, + + path_filestat_get__sig: 'iiiiii', + path_filestat_get: (fd, flags, path, path_len, buf) => { + return {{{ cDefs.ENOSYS }}}; + }, }; for (var x in WasiLibrary) { From 52a4e033066923cf746f8ee62ecac24eae7638a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 16:19:29 +0200 Subject: [PATCH 07/20] do preopen handling only in PURE_WASI mode --- system/lib/standalone/paths.c | 147 ++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/system/lib/standalone/paths.c b/system/lib/standalone/paths.c index 84e671287abe4..93f56d4a737d7 100644 --- a/system/lib/standalone/paths.c +++ b/system/lib/standalone/paths.c @@ -21,6 +21,80 @@ typedef struct preopen { /// A simple growable array of `preopen`. static preopen* preopens; static size_t num_preopens; + +/// Are the `prefix_len` bytes pointed to by `prefix` a prefix of `path`? +static bool +prefix_matches(const char* prefix, size_t prefix_len, const char* path) { + // Allow an empty string as a prefix of any relative path. + if (path[0] != '/' && prefix_len == 0) + return true; + + // Check whether any bytes of the prefix differ. + if (memcmp(path, prefix, prefix_len) != 0) + return false; + + // Ignore trailing slashes in directory names. + size_t i = prefix_len; + while (i > 0 && prefix[i - 1] == '/') { + --i; + } + + // Match only complete path components. + char last = path[i]; + return last == '/' || last == '\0'; +} + +bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr) { + const char* path = *path_ptr; + + if (*resolved_dirfd != AT_FDCWD && path[0] != '/') { + return true; + } + + // Strip leading `/` characters, the prefixes we're mataching won't have + // them. + while (*path == '/') + path++; + // Search through the preopens table. Iterate in reverse so that more + // recently added preopens take precedence over less recently addded ones. + size_t match_len = 0; + int fd = -1; + for (size_t i = num_preopens; i > 0; --i) { + const preopen* pre = &preopens[i - 1]; + const char* prefix = pre->prefix; + size_t len = strlen(prefix); + + // If we haven't had a match yet, or the candidate path is longer than + // our current best match's path, and the candidate path is a prefix of + // the requested path, take that as the new best path. + if ((fd == -1 || len > match_len) && prefix_matches(prefix, len, path)) { + fd = pre->fd; + match_len = len; + } + } + + if (fd == -1) { + return false; + } + + // The relative path is the substring after the portion that was matched. + const char* computed = path + match_len; + + // Omit leading slashes in the relative path. + while (*computed == '/') + ++computed; + + // *at syscalls don't accept empty relative paths, so use "." instead. + if (*computed == '\0') + computed = "."; + + *resolved_dirfd = fd; + *path_ptr = computed; + return true; +} + +#if defined(EMSCRIPTEN_PURE_WASI) + static size_t preopen_capacity; #ifdef NDEBUG @@ -112,77 +186,6 @@ static bool register_preopened_fd(__wasi_fd_t fd, const char* relprefix) { return true; } -/// Are the `prefix_len` bytes pointed to by `prefix` a prefix of `path`? -static bool -prefix_matches(const char* prefix, size_t prefix_len, const char* path) { - // Allow an empty string as a prefix of any relative path. - if (path[0] != '/' && prefix_len == 0) - return true; - - // Check whether any bytes of the prefix differ. - if (memcmp(path, prefix, prefix_len) != 0) - return false; - - // Ignore trailing slashes in directory names. - size_t i = prefix_len; - while (i > 0 && prefix[i - 1] == '/') { - --i; - } - - // Match only complete path components. - char last = path[i]; - return last == '/' || last == '\0'; -} - -bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr) { - const char* path = *path_ptr; - - if (*resolved_dirfd != AT_FDCWD && path[0] != '/') { - return true; - } - - // Strip leading `/` characters, the prefixes we're mataching won't have - // them. - while (*path == '/') - path++; - // Search through the preopens table. Iterate in reverse so that more - // recently added preopens take precedence over less recently addded ones. - size_t match_len = 0; - int fd = -1; - for (size_t i = num_preopens; i > 0; --i) { - const preopen* pre = &preopens[i - 1]; - const char* prefix = pre->prefix; - size_t len = strlen(prefix); - - // If we haven't had a match yet, or the candidate path is longer than - // our current best match's path, and the candidate path is a prefix of - // the requested path, take that as the new best path. - if ((fd == -1 || len > match_len) && prefix_matches(prefix, len, path)) { - fd = pre->fd; - match_len = len; - } - } - - if (fd == -1) { - return false; - } - - // The relative path is the substring after the portion that was matched. - const char* computed = path + match_len; - - // Omit leading slashes in the relative path. - while (*computed == '/') - ++computed; - - // *at syscalls don't accept empty relative paths, so use "." instead. - if (*computed == '\0') - computed = "."; - - *resolved_dirfd = fd; - *path_ptr = computed; - return true; -} - // Populate WASI preopens. __attribute__((constructor(100))) // construct this before user code static void _standalone_populate_preopens(void) { @@ -226,3 +229,5 @@ static void _standalone_populate_preopens(void) { software: _Exit(EX_SOFTWARE); } + +#endif From 53c309bec955e2745dd9941fd78fe5331f693b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 17:09:07 +0200 Subject: [PATCH 08/20] add path_filestat_get__nothrow: true --- src/lib/libwasi.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/libwasi.js b/src/lib/libwasi.js index 93783cd3b239f..01f44ed536899 100644 --- a/src/lib/libwasi.js +++ b/src/lib/libwasi.js @@ -614,6 +614,7 @@ var WasiLibrary = { }, path_filestat_get__sig: 'iiiiii', + path_filestat_get__nothrow: true, path_filestat_get: (fd, flags, path, path_len, buf) => { return {{{ cDefs.ENOSYS }}}; }, From 65e8c96e36b9996482ec524948c914129060e59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 17:46:21 +0200 Subject: [PATCH 09/20] run standalone WASM tests in a subdirectory of 'out/test' for each engine --- test/common.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test/common.py b/test/common.py index 238a9cec6c8f4..65ed695849e7d 100644 --- a/test/common.py +++ b/test/common.py @@ -1523,10 +1523,7 @@ def run_js(self, filename, engine=None, args=None, assert_returncode=0, interleaved_output=True, input=None, - run_in_tmpdir=False): - if run_in_tmpdir: - ensure_dir(self.in_dir('fs')) - + cwd=None): # use files, as PIPE can get too full and hang us stdout_file = self.in_dir('stdout') stderr_file = None @@ -1548,9 +1545,9 @@ def run_js(self, filename, engine=None, args=None, engine = engine + self.spidermonkey_args try: jsrun.run_js(filename, engine, args, - cwd=self.in_dir('fs') if run_in_tmpdir else None, stdout=stdout, stderr=stderr, + cwd=cwd, assert_returncode=assert_returncode, input=input) except subprocess.TimeoutExpired as e: @@ -1584,9 +1581,6 @@ def run_js(self, filename, engine=None, args=None, else: self.fail('JS subprocess failed (%s): %s (expected=%s). Output:\n%s' % (error.cmd, error.returncode, assert_returncode, ret)) - if run_in_tmpdir: - force_delete_contents(self.in_dir('fs')) - return ret def assertExists(self, filename, msg=None): @@ -1982,7 +1976,7 @@ def _build_and_run(self, filename, expected_output, args=None, js_file = self.build(filename, **kwargs) self.assertExists(js_file) - engines = [(el, False) for el in self.js_engines.copy()] + engines = [(el, None) for el in self.js_engines.copy()] if len(engines) > 1 and not self.use_all_engines: engines = engines[:1] # In standalone mode, also add wasm vms as we should be able to run there too. @@ -1991,14 +1985,17 @@ def _build_and_run(self, filename, expected_output, args=None, # like with js engines, but for now as we bring it up, test in all of them if not self.wasm_engines: logger.warning('no wasm engine was found to run the standalone part of this test') - engines += [(el, True) for el in self.wasm_engines] + engines += [(engine, os.path.basename(engine[0])) for engine in self.wasm_engines] if len(engines) == 0: self.fail('No JS engine present to run this test with. Check %s and the paths therein.' % config.EM_CONFIG) - for engine, run_in_tmpdir in engines: + for engine, subdir in engines: + if subdir: + subdir = self.in_dir(subdir) + ensure_dir(subdir) js_output = self.run_js(js_file, engine, args, + cwd=subdir, assert_returncode=assert_returncode, - interleaved_output=interleaved_output, - run_in_tmpdir=run_in_tmpdir) + interleaved_output=interleaved_output) js_output = js_output.replace('\r\n', '\n') if expected_output: if type(expected_output) not in [list, tuple]: From 5303f12d67698c310bf5f35fe0b7b29ee1928c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 19:30:04 +0200 Subject: [PATCH 10/20] try to fix CI --- src/lib/libsigs.js | 2 ++ src/lib/libwasi.js | 6 +++++- tools/building.py | 2 ++ tools/maint/gen_sig_info.py | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/libsigs.js b/src/lib/libsigs.js index c5474f4847718..4570960d52e7e 100644 --- a/src/lib/libsigs.js +++ b/src/lib/libsigs.js @@ -1536,6 +1536,8 @@ sigs = { lineColor__sig: 'ipiiiii', lineRGBA__sig: 'ipiiiiiiii', llvm_eh_typeid_for__sig: 'vp', + path_create_directory__sig: 'iipp', + path_filestat_get__sig: 'iiippp', pixelRGBA__sig: 'ipiiiiii', proc_exit__sig: 'vi', random_get__sig: 'ipp', diff --git a/src/lib/libwasi.js b/src/lib/libwasi.js index 01f44ed536899..4ba9c34fb4628 100644 --- a/src/lib/libwasi.js +++ b/src/lib/libwasi.js @@ -613,11 +613,15 @@ var WasiLibrary = { return 0; }, - path_filestat_get__sig: 'iiiiii', path_filestat_get__nothrow: true, path_filestat_get: (fd, flags, path, path_len, buf) => { return {{{ cDefs.ENOSYS }}}; }, + + path_create_directory__nothrow: true, + path_create_directory: (fd, path, path_len) => { + return {{{ cDefs.ENOSYS }}}; + }, }; for (var x in WasiLibrary) { diff --git a/tools/building.py b/tools/building.py index ab33a8602afa9..b768810586dab 100644 --- a/tools/building.py +++ b/tools/building.py @@ -819,6 +819,8 @@ def metadce(js_file, wasm_file, debug_info, last): 'clock_time_get', 'path_open', 'random_get', + 'path_filestat_get', + 'path_create_directory', } for item in graph: if 'import' in item and item['import'][1] in WASI_IMPORTS: diff --git a/tools/maint/gen_sig_info.py b/tools/maint/gen_sig_info.py index d8890da052749..d1cc215c5e140 100755 --- a/tools/maint/gen_sig_info.py +++ b/tools/maint/gen_sig_info.py @@ -162,6 +162,8 @@ 'args_get', 'args_sizes_get', 'random_get', + 'path_filestat_get', + 'path_create_directory', } From 95eab07ac71b7e099bad038107e3d4c6040181e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 20:50:58 +0200 Subject: [PATCH 11/20] refactor to avoid 'allow_wasm_engines' parameter --- test/common.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/common.py b/test/common.py index 65ed695849e7d..ad3bd5a3ed6bf 100644 --- a/test/common.py +++ b/test/common.py @@ -643,17 +643,19 @@ def metafunc(self, standalone): self.emcc_args.append('-Wno-unused-command-line-argument') # if we are impure, disallow all wasm engines if impure: - self.wasm_engines = [] + wasm_engines = [] else: - self.wasm_engines = [engine for engine in self.wasm_engines + wasm_engines = [engine for engine in self.wasm_engines if all(excluded not in os.path.basename(engine[0]) for excluded in exclude_engines)] if 'node' in exclude_engines: self.js_engines = [] if not self.wasm_engines: self.skipTest('no WASM engines available for this test') else: - nodejs = self.require_node(allow_wasm_engines=True) + nodejs = self.require_node() self.node_args += shared.node_bigint_flags(nodejs) + # `self.require_node()` clears `self.wasm_engines`, so set them here. + self.wasm_engines = wasm_engines func(self) parameterize(metafunc, {'': (False,), @@ -997,14 +999,14 @@ def get_nodejs(self): return None return config.NODE_JS_TEST - def require_node(self, allow_wasm_engines=False): + def require_node(self): nodejs = self.get_nodejs() if not nodejs: if 'EMTEST_SKIP_NODE' in os.environ: self.skipTest('test requires node and EMTEST_SKIP_NODE is set') else: self.fail('node required to run this test. Use EMTEST_SKIP_NODE to skip') - self.require_engine(nodejs, allow_wasm_engines) + self.require_engine(nodejs) return nodejs def node_is_canary(self, nodejs): @@ -1021,14 +1023,13 @@ def require_node_canary(self): else: self.fail('node canary required to run this test. Use EMTEST_SKIP_NODE_CANARY to skip') - def require_engine(self, engine, allow_wasm_engines=False): + def require_engine(self, engine): logger.debug(f'require_engine: {engine}') if self.required_engine and self.required_engine != engine: self.skipTest(f'Skipping test that requires `{engine}` when `{self.required_engine}` was previously required') self.required_engine = engine self.js_engines = [engine] - if not allow_wasm_engines: - self.wasm_engines = [] + self.wasm_engines = [] def require_wasm64(self): if self.is_browser_test(): From bd4a07c42831ebdcedbf819af8445133cd7742b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sat, 3 May 2025 23:27:13 +0200 Subject: [PATCH 12/20] remove hack by adding a stubbed out path_symlink to libwasi.js --- src/lib/libsigs.js | 1 + src/lib/libwasi.js | 5 +++++ test/common.py | 5 ----- tools/building.py | 1 + tools/maint/gen_sig_info.py | 1 + 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/lib/libsigs.js b/src/lib/libsigs.js index 4570960d52e7e..1c7c9e577bcb4 100644 --- a/src/lib/libsigs.js +++ b/src/lib/libsigs.js @@ -1538,6 +1538,7 @@ sigs = { llvm_eh_typeid_for__sig: 'vp', path_create_directory__sig: 'iipp', path_filestat_get__sig: 'iiippp', + path_symlink__sig: 'ippipp', pixelRGBA__sig: 'ipiiiiii', proc_exit__sig: 'vi', random_get__sig: 'ipp', diff --git a/src/lib/libwasi.js b/src/lib/libwasi.js index 4ba9c34fb4628..026c7cf7358ba 100644 --- a/src/lib/libwasi.js +++ b/src/lib/libwasi.js @@ -622,6 +622,11 @@ var WasiLibrary = { path_create_directory: (fd, path, path_len) => { return {{{ cDefs.ENOSYS }}}; }, + + path_symlink__nothrow: true, + path_symlink: (old_path, old_path_len, fd, new_path, new_path_len) => { + return {{{ cDefs.ENOSYS }}}; + }, }; for (var x in WasiLibrary) { diff --git a/test/common.py b/test/common.py index ad3bd5a3ed6bf..87823cf195290 100644 --- a/test/common.py +++ b/test/common.py @@ -631,11 +631,6 @@ def metafunc(self, standalone): self.set_setting('STANDALONE_WASM') if not impure: self.set_setting('PURE_WASI') - if 'node' in exclude_engines: - # When not running under node we don't care for any undefined symbols - # in the .js as we are only interested in the .wasm file. - self.set_setting('ERROR_ON_UNDEFINED_SYMBOLS=0') - self.emcc_args.append('-Wno-js-compiler') # we will not legalize the JS ffi interface, so we must use BigInt # support in order for JS to have a chance to run this without trapping # when it sees an i64 on the ffi. diff --git a/tools/building.py b/tools/building.py index b768810586dab..fc7e49b18a5f1 100644 --- a/tools/building.py +++ b/tools/building.py @@ -821,6 +821,7 @@ def metadce(js_file, wasm_file, debug_info, last): 'random_get', 'path_filestat_get', 'path_create_directory', + 'path_symlink', } for item in graph: if 'import' in item and item['import'][1] in WASI_IMPORTS: diff --git a/tools/maint/gen_sig_info.py b/tools/maint/gen_sig_info.py index d1cc215c5e140..a0e66c9af7a26 100755 --- a/tools/maint/gen_sig_info.py +++ b/tools/maint/gen_sig_info.py @@ -164,6 +164,7 @@ 'random_get', 'path_filestat_get', 'path_create_directory', + 'path_symlink', } From 36e66d8d5ece8df600c2e8fd73890dbc15a76eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 May 2025 10:11:27 +0200 Subject: [PATCH 13/20] add attribution to wasi-libc --- LICENSE | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/LICENSE b/LICENSE index 70cabe388d27d..c973e2ae71cb6 100644 --- a/LICENSE +++ b/LICENSE @@ -100,3 +100,31 @@ The third_party/ subdirectory contains code with other licenses. None of it is used by default, but certain options use it (e.g., the optional closure compiler flag will run closure compiler from third_party/). +Files in system/lib/standalone/ contain code derived from wasi-libc, in +accordance with the terms of the MIT license. wasi-libc's license follows: + + """ + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + """ From cc87f5d4bae802f7b448da16d963dd2204539826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 May 2025 10:35:31 +0200 Subject: [PATCH 14/20] should be wasm_engines instead of self.wasm_engines here --- test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index 87823cf195290..847c26df0257a 100644 --- a/test/common.py +++ b/test/common.py @@ -644,7 +644,7 @@ def metafunc(self, standalone): if all(excluded not in os.path.basename(engine[0]) for excluded in exclude_engines)] if 'node' in exclude_engines: self.js_engines = [] - if not self.wasm_engines: + if not wasm_engines: self.skipTest('no WASM engines available for this test') else: nodejs = self.require_node() From 67905b1dc20d4017ca32049e6954f86ee7981ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 17:37:58 +0200 Subject: [PATCH 15/20] add license headers and use pragma once for consistency --- system/lib/standalone/paths.c | 11 +++++++++++ system/lib/standalone/paths.h | 12 ++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/system/lib/standalone/paths.c b/system/lib/standalone/paths.c index 93f56d4a737d7..b18fcff7b1d66 100644 --- a/system/lib/standalone/paths.c +++ b/system/lib/standalone/paths.c @@ -1,3 +1,14 @@ +/* + * Copyright 2025 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + * + * The preopen code is based on wasi-libc's `preopens.c` which is licensed + * under a MIT style license. This license can also be found in the LICENSE + * file. + */ + #define _GNU_SOURCE #include "paths.h" diff --git a/system/lib/standalone/paths.h b/system/lib/standalone/paths.h index ceafd9da3b0e6..3e62d2fae1ec0 100644 --- a/system/lib/standalone/paths.h +++ b/system/lib/standalone/paths.h @@ -1,5 +1,11 @@ -#ifndef STANDALONE_PATHS_H -#define STANDALONE_PATHS_H +/* + * Copyright 2025 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#pragma once #include @@ -17,5 +23,3 @@ // Returns: `true` if resolution was successful, `false` otherwise. // bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr); - -#endif From f3a80bbc9aa2fe99154ab4ad582d385e4cb43b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 17:40:48 +0200 Subject: [PATCH 16/20] use double slash comments and braces for if statements consistently --- system/lib/standalone/paths.c | 48 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/system/lib/standalone/paths.c b/system/lib/standalone/paths.c index b18fcff7b1d66..c1fd7ef61a41a 100644 --- a/system/lib/standalone/paths.c +++ b/system/lib/standalone/paths.c @@ -20,29 +20,31 @@ #include #include -/// A name and file descriptor pair. +// A name and file descriptor pair. typedef struct preopen { - /// The path prefix associated with the file descriptor. + // The path prefix associated with the file descriptor. const char* prefix; - /// The file descriptor. + // The file descriptor. __wasi_fd_t fd; } preopen; -/// A simple growable array of `preopen`. +// A simple growable array of `preopen`. static preopen* preopens; static size_t num_preopens; -/// Are the `prefix_len` bytes pointed to by `prefix` a prefix of `path`? +// Are the `prefix_len` bytes pointed to by `prefix` a prefix of `path`? static bool prefix_matches(const char* prefix, size_t prefix_len, const char* path) { // Allow an empty string as a prefix of any relative path. - if (path[0] != '/' && prefix_len == 0) + if (path[0] != '/' && prefix_len == 0) { return true; + } // Check whether any bytes of the prefix differ. - if (memcmp(path, prefix, prefix_len) != 0) + if (memcmp(path, prefix, prefix_len) != 0) { return false; + } // Ignore trailing slashes in directory names. size_t i = prefix_len; @@ -64,8 +66,9 @@ bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr) { // Strip leading `/` characters, the prefixes we're mataching won't have // them. - while (*path == '/') + while (*path == '/') { path++; + } // Search through the preopens table. Iterate in reverse so that more // recently added preopens take precedence over less recently addded ones. size_t match_len = 0; @@ -92,12 +95,14 @@ bool __paths_resolve_path(int* resolved_dirfd, const char** path_ptr) { const char* computed = path + match_len; // Omit leading slashes in the relative path. - while (*computed == '/') + while (*computed == '/') { ++computed; + } // *at syscalls don't accept empty relative paths, so use "." instead. - if (*computed == '\0') + if (*computed == '\0') { computed = "."; + } *resolved_dirfd = fd; *path_ptr = computed; @@ -129,7 +134,7 @@ static void assert_invariants(void) { } #endif -/// Allocate space for more preopens. Returns 0 on success and -1 on failure. +// Allocate space for more preopens. Returns 0 on success and -1 on failure. static bool resize_preopens(void) { size_t start_capacity = 4; size_t old_capacity = preopen_capacity; @@ -170,9 +175,9 @@ static const char* strip_prefixes(const char* path) { return path; } -/// Register the given preopened file descriptor under the given path. -/// -/// This function takes ownership of `prefix`. +// Register the given preopened file descriptor under the given path. +// +// This function takes ownership of `prefix`. static bool register_preopened_fd(__wasi_fd_t fd, const char* relprefix) { // Check preconditions. assert_invariants(); @@ -205,26 +210,31 @@ static void _standalone_populate_preopens(void) { for (__wasi_fd_t fd = 3; fd != 0; ++fd) { __wasi_prestat_t prestat; __wasi_errno_t ret = __wasi_fd_prestat_get(fd, &prestat); - if (ret == __WASI_ERRNO_BADF) + if (ret == __WASI_ERRNO_BADF) { break; - if (ret != __WASI_ERRNO_SUCCESS) + } + if (ret != __WASI_ERRNO_SUCCESS) { goto oserr; + } switch (prestat.pr_type) { case __WASI_PREOPENTYPE_DIR: { char* prefix = malloc(prestat.u.dir.pr_name_len + 1); - if (prefix == NULL) + if (prefix == NULL) { goto software; + } // TODO: Remove the cast on `path` once the witx is updated with // char8 support. ret = __wasi_fd_prestat_dir_name( fd, (uint8_t*)prefix, prestat.u.dir.pr_name_len); - if (ret != __WASI_ERRNO_SUCCESS) + if (ret != __WASI_ERRNO_SUCCESS) { goto oserr; + } prefix[prestat.u.dir.pr_name_len] = '\0'; - if (!register_preopened_fd(fd, prefix)) + if (!register_preopened_fd(fd, prefix)) { goto software; + } free(prefix); break; From 534223f2922720e1a17cda2ce36b3c9ff171a41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 19:10:40 +0200 Subject: [PATCH 17/20] wrap new libwasi.js stubs in ALLOW_UNIMPLEMENTED_SYSCALLS --- src/lib/libwasi.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/libwasi.js b/src/lib/libwasi.js index 50058b269340b..e30ff08951ce7 100644 --- a/src/lib/libwasi.js +++ b/src/lib/libwasi.js @@ -609,6 +609,7 @@ var WasiLibrary = { return 0; }, +#if ALLOW_UNIMPLEMENTED_SYSCALLS path_filestat_get__nothrow: true, path_filestat_get: (fd, flags, path, path_len, buf) => { return {{{ cDefs.ENOSYS }}}; @@ -623,6 +624,7 @@ var WasiLibrary = { path_symlink: (old_path, old_path_len, fd, new_path, new_path_len) => { return {{{ cDefs.ENOSYS }}}; }, +#endif }; for (var x in WasiLibrary) { From 16d2ba75e7e0ccb9890ca96fcf591713eb75433d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 20:44:18 +0200 Subject: [PATCH 18/20] default 'exclude_engines' in argument list since we don't mutate it --- test/common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/common.py b/test/common.py index 847c26df0257a..2cd9245f36ec4 100644 --- a/test/common.py +++ b/test/common.py @@ -614,7 +614,7 @@ def can_do_standalone(self, impure=False): # Impure means a test that cannot run in a wasm VM yet, as it is not 100% # standalone. We can still run them with the JS code though. -def also_with_standalone_wasm(impure=False, exclude_engines=None): +def also_with_standalone_wasm(impure=False, exclude_engines=[]): # noqa: B006 def decorated(func): @wraps(func) def metafunc(self, standalone): @@ -623,9 +623,6 @@ def metafunc(self, standalone): if not standalone: func(self) else: - nonlocal exclude_engines - if exclude_engines is None: - exclude_engines = [] if not can_do_standalone(self, impure): self.skipTest('Test configuration is not compatible with STANDALONE_WASM') self.set_setting('STANDALONE_WASM') From eb35b6bcdc7f3b2f2cc95f8ed2d2cfa7449eeb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 20:53:53 +0200 Subject: [PATCH 19/20] simplify calculation of final 'wasm_engines' --- test/common.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/common.py b/test/common.py index 2cd9245f36ec4..94278d9ff806b 100644 --- a/test/common.py +++ b/test/common.py @@ -633,12 +633,13 @@ def metafunc(self, standalone): # when it sees an i64 on the ffi. self.set_setting('WASM_BIGINT') self.emcc_args.append('-Wno-unused-command-line-argument') + wasm_engines = [] # if we are impure, disallow all wasm engines - if impure: - wasm_engines = [] - else: - wasm_engines = [engine for engine in self.wasm_engines - if all(excluded not in os.path.basename(engine[0]) for excluded in exclude_engines)] + if not impure: + for engine in self.wasm_engines: + basename = os.path.basename(engine[0]) + if not any(pattern in basename for pattern in exclude_engines): + wasm_engines.append(engine) if 'node' in exclude_engines: self.js_engines = [] if not wasm_engines: From 1ec092a7e3e9398def61ebb20e50441ea07f5e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Fri, 9 May 2025 20:54:12 +0200 Subject: [PATCH 20/20] add comment explaining 'exclude_engines' parameter --- test/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/common.py b/test/common.py index 94278d9ff806b..81eeadff96920 100644 --- a/test/common.py +++ b/test/common.py @@ -614,6 +614,9 @@ def can_do_standalone(self, impure=False): # Impure means a test that cannot run in a wasm VM yet, as it is not 100% # standalone. We can still run them with the JS code though. +# "exclude_engines" is a list of engine names on which the test cannot run. +# These are pattern that are matched against the basename of the engine +# executable. def also_with_standalone_wasm(impure=False, exclude_engines=[]): # noqa: B006 def decorated(func): @wraps(func)