From da18e4446126d75588bd4b4f9c6ec626755d2d6b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 11:28:54 -0700 Subject: [PATCH 01/49] main --- tools/wasm2c/main.c | 602 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 602 insertions(+) create mode 100644 tools/wasm2c/main.c diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c new file mode 100644 index 0000000000000..6265333f16acd --- /dev/null +++ b/tools/wasm2c/main.c @@ -0,0 +1,602 @@ +/* + * A main file to run wasm2c code. This implements various wasi and emscripten + * syscalls, and allows direct/unsandboxed file access TODO add options + */ + +#define __USE_GNU // for O_PATH + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "wasm-rt.h" +#include "wasm-rt-impl.h" + +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#define LIKELY(x) __builtin_expect(!!(x), 1) + +#define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0) + +#define MEMACCESS(addr) ((void*)&Z_memory->data[addr]) + +#define MEMCHECK(a, t) \ + if (UNLIKELY((a) + sizeof(t) > Z_memory->size)) TRAP(OOB) + +#define DEFINE_LOAD(name, t1, t2, t3) \ + static inline t3 name(u64 addr) { \ + MEMCHECK(addr, t1); \ + t1 result; \ + memcpy(&result, MEMACCESS(addr), sizeof(t1)); \ + return (t3)(t2)result; \ + } + +#define DEFINE_STORE(name, t1, t2) \ + static inline void name(u64 addr, t2 value) { \ + MEMCHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + memcpy(MEMACCESS(addr), &wrapped, sizeof(t1)); \ + } + +DEFINE_LOAD(i32_load, u32, u32, u32); +DEFINE_LOAD(i64_load, u64, u64, u64); +DEFINE_LOAD(f32_load, f32, f32, f32); +DEFINE_LOAD(f64_load, f64, f64, f64); +DEFINE_LOAD(i32_load8_s, s8, s32, u32); +DEFINE_LOAD(i64_load8_s, s8, s64, u64); +DEFINE_LOAD(i32_load8_u, u8, u32, u32); +DEFINE_LOAD(i64_load8_u, u8, u64, u64); +DEFINE_LOAD(i32_load16_s, s16, s32, u32); +DEFINE_LOAD(i64_load16_s, s16, s64, u64); +DEFINE_LOAD(i32_load16_u, u16, u32, u32); +DEFINE_LOAD(i64_load16_u, u16, u64, u64); +DEFINE_LOAD(i64_load32_s, s32, s64, u64); +DEFINE_LOAD(i64_load32_u, u32, u64, u64); +DEFINE_STORE(i32_store, u32, u32); +DEFINE_STORE(i64_store, u64, u64); +DEFINE_STORE(f32_store, f32, f32); +DEFINE_STORE(f64_store, f64, f64); +DEFINE_STORE(i32_store8, u8, u32); +DEFINE_STORE(i32_store16, u16, u32); +DEFINE_STORE(i64_store8, u8, u64); +DEFINE_STORE(i64_store16, u16, u64); +DEFINE_STORE(i64_store32, u32, u64); + +// Imports + +#ifdef VERBOSE_LOGGING +#define VERBOSE_LOG(...) { printf(__VA_ARGS__); } +#else +#define VERBOSE_LOG(...) +#endif + +#define IMPORT_IMPL(ret, name, params, body) \ +ret _##name params { \ + VERBOSE_LOG("[import: " #name "]\n"); \ + body \ +} \ +ret (*name) params = _##name; + +#define STUB_IMPORT_IMPL(ret, name, params, returncode) IMPORT_IMPL(ret, name, params, { return returncode; }); + +#define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ + +IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { + exit(x); +}); + +#define MAX_FDS 1024 + +static int wasm_fd_to_native[MAX_FDS]; + +static u32 next_wasm_fd; + +static void init_fds() { + wasm_fd_to_native[0] = STDIN_FILENO; + wasm_fd_to_native[1] = STDOUT_FILENO; + wasm_fd_to_native[2] = STDERR_FILENO; + next_wasm_fd = 3; +} + +void abort_with_message(const char* message) { + fprintf(stderr, "%s\n", message); + abort(); +} + +static u32 get_or_allocate_wasm_fd(int nfd) { + // If the native fd is already mapped, return the same wasm fd for it. + for (int i = 0; i < next_wasm_fd; i++) { + if (wasm_fd_to_native[i] == nfd) { + return i; + } + } + if (next_wasm_fd >= MAX_FDS) { + abort_with_message("ran out of fds"); + } + u32 fd = next_wasm_fd; + wasm_fd_to_native[fd] = nfd; + next_wasm_fd++; + return fd; +} + +static int get_native_fd(u32 fd) { + if (fd >= MAX_FDS || fd >= next_wasm_fd) { + return -1; + } + return wasm_fd_to_native[fd]; +} + +IMPORT_IMPL(u32, Z_envZ___sys_openZ_iiii, (u32 path, u32 flags, u32 varargs), { + VERBOSE_LOG(" open: %s %d %d\n", MEMACCESS(path), flags, i32_load(varargs)); + int nfd = open(MEMACCESS(path), flags, i32_load(varargs)); + VERBOSE_LOG(" => native %d\n", nfd); + if (nfd >= 0) { + u32 fd = get_or_allocate_wasm_fd(nfd); + VERBOSE_LOG(" => wasm %d\n", fd); + return fd; + } + return -1; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fd_write wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + u32 num = 0; + for (u32 i = 0; i < iovcnt; i++) { + u32 ptr = i32_load(iov + i * 8); + u32 len = i32_load(iov + i * 8 + 4); + VERBOSE_LOG(" chunk %d %d\n", ptr, len); + ssize_t result; + // Use stdio for stdout/stderr to avoid mixing a low-level write() with + // other logging code, which can change the order from the expected. + if (nfd == STDOUT_FILENO) { + result = fwrite(MEMACCESS(ptr), 1, len, stdout); + } else if (nfd == STDERR_FILENO) { + result = fwrite(MEMACCESS(ptr), 1, len, stderr); + } else { + result = write(nfd, MEMACCESS(ptr), len); + } + if (result < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + if (result != len) { + VERBOSE_LOG(" amount error, %ld %d\n", result, len); + return WASI_DEFAULT_ERROR; + } + num += len; + } + VERBOSE_LOG(" success: %d\n", num); + i32_store(pnum, num); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fd_read wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + u32 num = 0; + for (u32 i = 0; i < iovcnt; i++) { + u32 ptr = i32_load(iov + i * 8); + u32 len = i32_load(iov + i * 8 + 4); + VERBOSE_LOG(" chunk %d %d\n", ptr, len); + ssize_t result = read(nfd, MEMACCESS(ptr), len); + if (result < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + num += result; + if (result != len) { + break; // nothing more to read + } + } + VERBOSE_LOG(" success: %d\n", num); + i32_store(pnum, num); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_closeZ_ii, (u32 fd), { + // TODO full file support + int nfd = get_native_fd(fd); + VERBOSE_LOG(" close wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + close(nfd); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_sizes_getZ_iii, (u32 pcount, u32 pbuf_size), { + // TODO: connect to actual env? + i32_store(pcount, 0); + i32_store(pbuf_size, 0); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_getZ_iii, (u32 __environ, u32 environ_buf), { + // TODO: connect to actual env? + return 0; +}); + +static int whence_to_native(u32 whence) { + if (whence == 0) return SEEK_SET; + if (whence == 1) return SEEK_CUR; + if (whence == 2) return SEEK_END; + return -1; +} + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, u32 whence, u32 new_offset), { + int nfd = get_native_fd(fd); + int nwhence = whence_to_native(whence); + VERBOSE_LOG(" seek %d (=> native %d) %ld %d (=> %d) %d\n", fd, nfd, offset, whence, nwhence, new_offset); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + off_t off = lseek(nfd, offset, nwhence); + VERBOSE_LOG(" off: %ld\n", off); + if (off == (off_t)-1) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + i64_store(new_offset, off); + return 0; +}); +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), { + return Z_wasi_snapshot_preview1Z_fd_seekZ_iijii(a, b + (((u64)c) << 32), d, e); +}); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_fdstat_getZ_iii, (u32 a, u32 b), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_syncZ_ii, (u32 a), WASI_DEFAULT_ERROR); + +// TODO: set errno in wasm for everything + +STUB_IMPORT_IMPL(u32, Z_envZ_dlopenZ_iii, (u32 a, u32 b), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlcloseZ_ii, (u32 a), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlsymZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_dlerrorZ_iv, (), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_signalZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_systemZ_ii, (u32 a), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_utimesZ_iii, (u32 a, u32 b), -1); + +// Syscalls return a negative error code +#define EM_EACCES -2 + +IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), { + VERBOSE_LOG(" unlink %s\n", MEMACCESS(path)); + if (unlink(MEMACCESS(path))) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return 0; +}); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_rmdirZ_ii, (u32 a), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_renameZ_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_lstat64Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup3Z_iiii, (u32 a, u32 b, u32 c), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup2Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_getcwdZ_iii, (u32 a, u32 b), EM_EACCES); + +static u32 do_stat(int nfd, u32 buf) { + struct stat nbuf; + if (fstat(nfd, &nbuf)) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + VERBOSE_LOG(" success, size=%ld\n", nbuf.st_size); + i32_store(buf + 0, nbuf.st_dev); + i32_store(buf + 4, 0); + i32_store(buf + 8, nbuf.st_ino); + i32_store(buf + 12, nbuf.st_mode); + i32_store(buf + 16, nbuf.st_nlink); + i32_store(buf + 20, nbuf.st_uid); + i32_store(buf + 24, nbuf.st_gid); + i32_store(buf + 28, nbuf.st_rdev); + i32_store(buf + 32, 0); + i64_store(buf + 40, nbuf.st_size); + i32_store(buf + 48, nbuf.st_blksize); + i32_store(buf + 52, nbuf.st_blocks); + i32_store(buf + 56, nbuf.st_atim.tv_sec); + i32_store(buf + 60, nbuf.st_atim.tv_nsec); + i32_store(buf + 64, nbuf.st_mtim.tv_sec); + i32_store(buf + 68, nbuf.st_mtim.tv_nsec); + i32_store(buf + 72, nbuf.st_ctim.tv_sec); + i32_store(buf + 76, nbuf.st_ctim.tv_nsec); + i64_store(buf + 80, nbuf.st_ino); + return 0; +} + +IMPORT_IMPL(u32, Z_envZ___sys_fstat64Z_iii, (u32 fd, u32 buf), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fstat64 %d (=> %d) %d\n", fd, nfd, buf); + if (nfd < 0) { + return EM_EACCES; + } + return do_stat(nfd, buf); +}); + +IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), { + VERBOSE_LOG(" stat64: %s\n", MEMACCESS(path)); + int nfd = open(MEMACCESS(path), O_PATH); + if (nfd < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return do_stat(nfd, buf); +}); + +STUB_IMPORT_IMPL(u32, Z_envZ___sys_ftruncate64Z_iiiii, (u32 a, u32 b, u32 c, u32 d), EM_EACCES); +IMPORT_IMPL(u32, Z_envZ___sys_readZ_iiii, (u32 fd, u32 buf, u32 count), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" read %d (=> %d) %d %d\n", fd, nfd, buf, count); + if (nfd < 0) { + VERBOSE_LOG(" bad fd\n"); + return EM_EACCES; + } + ssize_t ret = read(nfd, MEMACCESS(buf), count); + VERBOSE_LOG(" native read: %ld\n", ret); + if (ret < 0) { + VERBOSE_LOG(" read error %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return ret; +}); + +IMPORT_IMPL(u32, Z_envZ___sys_accessZ_iii, (u32 pathname, u32 mode), { + VERBOSE_LOG(" access: %s 0x%x\n", MEMACCESS(pathname), mode); + // TODO: sandboxing, convert mode + int result = access(MEMACCESS(pathname), mode); + if (result < 0) { + VERBOSE_LOG(" access error: %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return 0; +}); + +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_initZ_ii, (u32 a), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_settypeZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_destroyZ_ii, (u32 a), 0); + +static int main_argc; +static char** main_argv; + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_args_sizes_getZ_iii, (u32 pargc, u32 pargv_buf_size), { + i32_store(pargc, main_argc); + u32 buf_size = 0; + for (u32 i = 0; i < main_argc; i++) { + buf_size += strlen(main_argv[i]) + 1; + } + i32_store(pargv_buf_size, buf_size); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_args_getZ_iii, (u32 argv, u32 argv_buf), { + u32 buf_size = 0; + for (u32 i = 0; i < main_argc; i++) { + u32 ptr = argv_buf + buf_size; + i32_store(argv + i * 4, ptr); + char* arg = main_argv[i]; + strcpy(MEMACCESS(ptr), arg); + buf_size += strlen(arg) + 1; + } + return 0; +}); + +// Maintain a stack of setjmps, each jump taking us back to the last invoke. + +#define MAX_SETJMP_STACK 1024 + +static jmp_buf setjmp_stack[MAX_SETJMP_STACK]; + +static u32 next_setjmp = 0; + +// Declare exports for invokes. We should generate them based on what the +// wasm needs, but for now have a fixed list here. To get things to link, +// declare them, so they either link with the existing value in the main +// wasm2c .c output file, or else they contain NULL but will never be called. + +#define DECLARE_EXPORT(ret, name, args) \ +__attribute__((weak)) \ +ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; + +DECLARE_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); + +#define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ +DECLARE_EXPORT(void, dyncall, types); \ +\ +IMPORT_IMPL(void, name, typed_args, { \ + VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ + u32 sp = Z_stackSaveZ_iv(); \ + if (next_setjmp >= MAX_SETJMP_STACK) { \ + abort_with_message("too many nested setjmps"); \ + } \ + u32 id = next_setjmp++; \ + int result = setjmp(setjmp_stack[id]); \ + if (result == 0) { \ + (* dyncall) args; \ + /* if we got here, no longjmp or exception happened, we returned normally */ \ + } else { \ + /* A longjmp or an exception took us here. */ \ + Z_stackRestoreZ_vi(sp); \ + Z_setThrewZ_vii(1, 0); \ + } \ + next_setjmp--; \ +}); + +#define RETURNING_INVOKE_IMPL(ret, name, typed_args, types, args, dyncall) \ +DECLARE_EXPORT(ret, dyncall, types); \ +\ +IMPORT_IMPL(ret, name, typed_args, { \ + VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ + u32 sp = Z_stackSaveZ_iv(); \ + if (next_setjmp >= MAX_SETJMP_STACK) { \ + abort_with_message("too many nested setjmps"); \ + } \ + u32 id = next_setjmp++; \ + int result = setjmp(setjmp_stack[id]); \ + ret returned_value = 0; \ + if (result == 0) { \ + returned_value = (* dyncall) args; \ + /* if we got here, no longjmp or exception happened, we returned normally */ \ + } else { \ + /* A longjmp or an exception took us here. */ \ + Z_stackRestoreZ_vi(sp); \ + Z_setThrewZ_vii(1, 0); \ + } \ + next_setjmp--; \ + return returned_value; \ +}); + +VOID_INVOKE_IMPL(Z_envZ_invoke_vZ_vi, (u32 fptr), (u32), (fptr), Z_dynCall_vZ_vi); +VOID_INVOKE_IMPL(Z_envZ_invoke_viiZ_viii, (u32 fptr, u32 a, u32 b), (u32, u32, u32), (fptr, a, b), Z_dynCall_viiZ_viii); +VOID_INVOKE_IMPL(Z_envZ_invoke_viiiZ_viiii, (u32 fptr, u32 a, u32 b, u32 c), (u32, u32, u32, u32), (fptr, a, b, c), Z_dynCall_viiiZ_viiii); + +RETURNING_INVOKE_IMPL(u32, Z_envZ_invoke_iiiZ_iiii, (u32 fptr, u32 a, u32 b), (u32, u32, u32), (fptr, a, b), Z_dynCall_iiiZ_iiii); +RETURNING_INVOKE_IMPL(u32, Z_envZ_invoke_iiZ_iii, (u32 fptr, u32 a), (u32, u32), (fptr, a), Z_dynCall_iiZ_iii); + +IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { + if (next_setjmp == 0) { + abort_with_message("longjmp without setjmp"); + } + Z_setThrewZ_vii(buf, value ? value : 1); + longjmp(setjmp_stack[next_setjmp - 1], 1); +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { + // TODO: handle realtime vs monotonic etc. + // wasi expects a result in nanoseconds, and we know how to convert clock() + // to seconds, so compute from there + const double NSEC_PER_SEC = 1000.0 * 1000.0 * 1000.0; + i64_store(out, (u64)(clock() / (CLOCKS_PER_SEC / NSEC_PER_SEC))); + return 0; +}); + +IMPORT_IMPL(void, Z_envZ_emscripten_notify_memory_growthZ_vi, (u32 size), {}); + +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_joinZ_iii, (u32 a, u32 b), -1); + +STUB_IMPORT_IMPL(u32, Z_envZ___cxa_thread_atexitZ_iiii, (u32 a, u32 b, u32 c), -1); + +static u32 tempRet0 = 0; + +IMPORT_IMPL(u32, Z_envZ_getTempRet0Z_iv, (), { + return tempRet0; +}); + +IMPORT_IMPL(void, Z_envZ_setTempRet0Z_vi, (u32 x), { + tempRet0 = x; +}); + +// autodebug + +IMPORT_IMPL(void, Z_envZ_log_executionZ_vi, (u32 loc), { + printf("log_execution %d\n", loc); +}); +IMPORT_IMPL(u32, Z_envZ_get_i32Z_iiii, (u32 loc, u32 index, u32 value), { + printf("get_i32 %d,%d,%d\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_get_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { + printf("get_i64 %d,%d,%d,%d\n", loc, index, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_get_f32Z_fiif, (u32 loc, u32 index, f32 value), { + printf("get_f32 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_get_f64Z_diid, (u32 loc, u32 index, f64 value), { + printf("get_f64 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_set_i32Z_iiii, (u32 loc, u32 index, u32 value), { + printf("set_i32 %d,%d,%d\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_set_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { + printf("set_i64 %d,%d,%d,%d\n", loc, index, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_set_f32Z_fiif, (u32 loc, u32 index, f32 value), { + printf("set_f32 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_set_f64Z_diid, (u32 loc, u32 index, f64 value), { + printf("set_f64 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_load_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { + printf("load_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); + return ptr; +}); +IMPORT_IMPL(u32, Z_envZ_load_val_i32Z_iii, (u32 loc, u32 value), { + printf("load_val_i32 %d,%d\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_load_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { + printf("load_val_i64 %d,%d,%d\n", loc, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_load_val_f32Z_fif, (u32 loc, f32 value), { + printf("load_val_f32 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_load_val_f64Z_did, (u32 loc, f64 value), { + printf("load_val_f64 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_store_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { + printf("store_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); + return ptr; +}); +IMPORT_IMPL(u32, Z_envZ_store_val_i32Z_iii, (u32 loc, u32 value), { + printf("store_val_i32 %d,%d\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_store_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { + printf("store_val_i64 %d,%d,%d\n", loc, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_store_val_f32Z_fif, (u32 loc, f32 value), { + printf("store_val_f32 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_store_val_f64Z_did, (u32 loc, f64 value), { + printf("store_val_f64 %d,%f\n", loc, value); + return value; +}); + +// Main + +int main(int argc, char** argv) { + main_argc = argc; + main_argv = argv; + + init_fds(); + + init(); + + int trap_code; + if ((trap_code = setjmp(g_jmp_buf))) { + printf("[wasm trap %d, halting]\n", trap_code); + return 1; + } else { + Z__startZ_vv(); + } + return 0; +} From c208a6dbec4f205f375a6bac0b368103f1c29028 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 11:31:50 -0700 Subject: [PATCH 02/49] start --- tests/test_benchmark.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 3a7f763e8849a..8fbe29fa13073 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -239,6 +239,41 @@ def cleanup(self): Building.clear() +class EmscriptenWasm2CBenchmarker(EmscriptenBenchmarker): + def __init__(self, name): + super(EmscriptenWasm2CBenchmarker, self).__init__(name, 'no engine needed') + + def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): + # wasm2c doesn't want minimal runtime which the normal emscripten + # benchmarker defaults to, as we don't have any JS anyhow + emcc_args = emcc_args + ['-s', 'STANDALONE_WASM', '-s', 'MINIMAL_RUNTIME=0'] + + super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) + + base = self.filename[:-3] + wasm = base + '.wasm' + c = base + '.c' + h = base + '.h' + native = base + '.exe' + + wabt_dir = os.path.expanduser('~/Dev/wabt/') + run_process([os.path.join(wabt_dir, 'build', 'wasm2c'), wasm, '-o', c]) + run_process(['clang', os.path.join(wabt_dir, 'wasm2c', 'main-emscripten.c'), c, + os.path.join(wabt_dir, 'wasm2c', 'wasm-rt-impl.c'), '-I.', + '-I' + os.path.join(wabt_dir, 'wasm2c'), '-lm', + '-include', h, '-o', native, OPTIMIZATIONS, + '-DWASM_RT_MAX_CALL_STACK_DEPTH=8000']) # for havlak + + self.filename = native + + def run(self, args): + return run_process([self.filename] + args, stdout=PIPE, stderr=subprocess.STDOUT, check=False).stdout + + def get_output_files(self): + # return the native code. c size may also be interesting. + return [self.filename] + + CHEERP_BIN = '/opt/cheerp/bin/' From 802b60ce0d28c9c015cdb33758134e092f0c3368 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 11:32:42 -0700 Subject: [PATCH 03/49] more [ci skip] --- tests/test_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 8fbe29fa13073..21ea338ccfe32 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -246,7 +246,7 @@ def __init__(self, name): def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): # wasm2c doesn't want minimal runtime which the normal emscripten # benchmarker defaults to, as we don't have any JS anyhow - emcc_args = emcc_args + ['-s', 'STANDALONE_WASM', '-s', 'MINIMAL_RUNTIME=0'] + emcc_args = emcc_args + ['-s', 'STANDALONE_WASM', '-s', 'MINIMAL_RUNTIME=0'] # WASM2C flag? super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) From 2daf554e69b0d15434c3ab15a468c84f071e05cb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 18:11:43 -0700 Subject: [PATCH 04/49] wip [ci skip] --- emcc.py | 5 +++++ src/settings.js | 9 +++++++++ tools/shared.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/emcc.py b/emcc.py index 9c9531f30d3cd..96253585b5f3a 100755 --- a/emcc.py +++ b/emcc.py @@ -3338,6 +3338,11 @@ def run_closure_compiler(final): dwarf_target = wasm_binary_target + '.debug.wasm' shared.Building.emit_debug_on_side(wasm_binary_target, dwarf_target) + if shared.Settings.WASM2C: + # FIXME this assumes the emsdk layout where the binaryen and wabt binaries + # are together + shared.Building.do_wasm2c(wasm_binary_target) + # replace placeholder strings with correct subresource locations if shared.Settings.SINGLE_FILE: js = open(final).read() diff --git a/src/settings.js b/src/settings.js index c5df7de6f3491..8f263f04264db 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1753,6 +1753,15 @@ var DEFAULT_TO_CXX = 1; // long double printing precision. var PRINTF_LONG_DOUBLE = 0; +// Run wabt's wasm2c tool on the final wasm, and combine that with a C runtime, +// resulting in a .c file that you can compile with a C compiler to get a +// native executable that works the same as the normal js+wasm. +// When using this you must specify WABT_BIN, which should be where the wasm2c +// executable can be found. We will also look for the wasm2c directory near it +// as we need headers and other support there. +var WASM2C = 0; +var WABT_BIN = ''; + //=========================================== // Internal, used for testing only, from here //=========================================== diff --git a/tools/shared.py b/tools/shared.py index 92b559a09a238..6390099592ba6 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2782,6 +2782,40 @@ def run_binaryen_command(tool, infile, outfile=None, args=[], debug=False, stdou def run_wasm_opt(*args, **kwargs): return Building.run_binaryen_command('wasm-opt', *args, **kwargs) + @staticmethod + def do_wasm2c(infile): + # look for the wasm2c/ dir alongside the bin dir, or perhaps higher up. + WASM2C_DIR = os.path.dirname(Settings.WABT_BIN) + while WASM2C_DIR and not os.path.exists(os.path.join(WASM2C_DIR, 'wasm2c')): + WASM2C_DIR = os.path.dirname(WASM2C_DIR) + if not WASM2C_DIR: + exit_with_error('Could not find wabt wasm2c/ dir in the tree under ' + Settings.WABT_BIN) + WASM2C_DIR = os.path.join(WASM2C_DIR, 'wasm2c') + c_file = unsuffixed(infile) + '.c' + h_file = unsuffixed(infile) + '.h' + cmd = [os.path.join(Settings.WABT_BIN, 'wasm2c'), infile, '-o', c_file] + run_process(cmd) + with open(c_file) as read_c: + c = read_c.read() + SEP = '\n//====================\n\n' + # hermeticize the C file, by bundling in the wasm2c/ includes + headers = [ + (WASM2C_DIR, 'wasm-rt.h'), + (WASM2C_DIR, 'wasm-rt-impl.h'), + (os.path.dirname(h_file), os.path.basename(h_file)) + ] + total = '' + for header in headers: + with open(os.path.join(header[0], header[1])) as f: + total += f.read() + SEP + total += c + SEP + with open(path_from_root('tools', 'wasm2c', 'main.c')) as main: + total += main.read() + for header in headers: + total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) + with open(c_file, 'w') as out: + out.write(total) + save_intermediate_counter = 0 @staticmethod From 7c0c9208a061661cf792b3bfa1bcd952d6cc9391 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 20:00:18 -0700 Subject: [PATCH 05/49] fixes [ci skip] --- tools/shared.py | 2 + tools/wasm2c/main.c | 118 ++++++++++++++++++++++---------------------- 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 6390099592ba6..c895e6cf58c6e 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2809,6 +2809,8 @@ def do_wasm2c(infile): with open(os.path.join(header[0], header[1])) as f: total += f.read() + SEP total += c + SEP + with open(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) as f: + total += f.read() + SEP with open(path_from_root('tools', 'wasm2c', 'main.c')) as main: total += main.read() for header in headers: diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 6265333f16acd..1bbd0525b4451 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -26,10 +26,10 @@ #define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0) -#define MEMACCESS(addr) ((void*)&Z_memory->data[addr]) +#define MEMACCESS(addr) ((void*)&Z_envZ_memory->data[addr]) #define MEMCHECK(a, t) \ - if (UNLIKELY((a) + sizeof(t) > Z_memory->size)) TRAP(OOB) + if (UNLIKELY((a) + sizeof(t) > Z_envZ_memory->size)) TRAP(OOB) #define DEFINE_LOAD(name, t1, t2, t3) \ static inline t3 name(u64 addr) { \ @@ -46,29 +46,29 @@ memcpy(MEMACCESS(addr), &wrapped, sizeof(t1)); \ } -DEFINE_LOAD(i32_load, u32, u32, u32); -DEFINE_LOAD(i64_load, u64, u64, u64); -DEFINE_LOAD(f32_load, f32, f32, f32); -DEFINE_LOAD(f64_load, f64, f64, f64); -DEFINE_LOAD(i32_load8_s, s8, s32, u32); -DEFINE_LOAD(i64_load8_s, s8, s64, u64); -DEFINE_LOAD(i32_load8_u, u8, u32, u32); -DEFINE_LOAD(i64_load8_u, u8, u64, u64); -DEFINE_LOAD(i32_load16_s, s16, s32, u32); -DEFINE_LOAD(i64_load16_s, s16, s64, u64); -DEFINE_LOAD(i32_load16_u, u16, u32, u32); -DEFINE_LOAD(i64_load16_u, u16, u64, u64); -DEFINE_LOAD(i64_load32_s, s32, s64, u64); -DEFINE_LOAD(i64_load32_u, u32, u64, u64); -DEFINE_STORE(i32_store, u32, u32); -DEFINE_STORE(i64_store, u64, u64); -DEFINE_STORE(f32_store, f32, f32); -DEFINE_STORE(f64_store, f64, f64); -DEFINE_STORE(i32_store8, u8, u32); -DEFINE_STORE(i32_store16, u16, u32); -DEFINE_STORE(i64_store8, u8, u64); -DEFINE_STORE(i64_store16, u16, u64); -DEFINE_STORE(i64_store32, u32, u64); +DEFINE_LOAD(wasm_i32_load, u32, u32, u32); +DEFINE_LOAD(wasm_i64_load, u64, u64, u64); +DEFINE_LOAD(wasm_f32_load, f32, f32, f32); +DEFINE_LOAD(wasm_f64_load, f64, f64, f64); +DEFINE_LOAD(wasm_i32_load8_s, s8, s32, u32); +DEFINE_LOAD(wasm_i64_load8_s, s8, s64, u64); +DEFINE_LOAD(wasm_i32_load8_u, u8, u32, u32); +DEFINE_LOAD(wasm_i64_load8_u, u8, u64, u64); +DEFINE_LOAD(wasm_i32_load16_s, s16, s32, u32); +DEFINE_LOAD(wasm_i64_load16_s, s16, s64, u64); +DEFINE_LOAD(wasm_i32_load16_u, u16, u32, u32); +DEFINE_LOAD(wasm_i64_load16_u, u16, u64, u64); +DEFINE_LOAD(wasm_i64_load32_s, s32, s64, u64); +DEFINE_LOAD(wasm_i64_load32_u, u32, u64, u64); +DEFINE_STORE(wasm_i32_store, u32, u32); +DEFINE_STORE(wasm_i64_store, u64, u64); +DEFINE_STORE(wasm_f32_store, f32, f32); +DEFINE_STORE(wasm_f64_store, f64, f64); +DEFINE_STORE(wasm_i32_store8, u8, u32); +DEFINE_STORE(wasm_i32_store16, u16, u32); +DEFINE_STORE(wasm_i64_store8, u8, u64); +DEFINE_STORE(wasm_i64_store16, u16, u64); +DEFINE_STORE(wasm_i64_store32, u32, u64); // Imports @@ -135,8 +135,8 @@ static int get_native_fd(u32 fd) { } IMPORT_IMPL(u32, Z_envZ___sys_openZ_iiii, (u32 path, u32 flags, u32 varargs), { - VERBOSE_LOG(" open: %s %d %d\n", MEMACCESS(path), flags, i32_load(varargs)); - int nfd = open(MEMACCESS(path), flags, i32_load(varargs)); + VERBOSE_LOG(" open: %s %d %d\n", MEMACCESS(path), flags, wasm_i32_load(varargs)); + int nfd = open(MEMACCESS(path), flags, wasm_i32_load(varargs)); VERBOSE_LOG(" => native %d\n", nfd); if (nfd >= 0) { u32 fd = get_or_allocate_wasm_fd(nfd); @@ -154,8 +154,8 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u3 } u32 num = 0; for (u32 i = 0; i < iovcnt; i++) { - u32 ptr = i32_load(iov + i * 8); - u32 len = i32_load(iov + i * 8 + 4); + u32 ptr = wasm_i32_load(iov + i * 8); + u32 len = wasm_i32_load(iov + i * 8 + 4); VERBOSE_LOG(" chunk %d %d\n", ptr, len); ssize_t result; // Use stdio for stdout/stderr to avoid mixing a low-level write() with @@ -178,7 +178,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u3 num += len; } VERBOSE_LOG(" success: %d\n", num); - i32_store(pnum, num); + wasm_i32_store(pnum, num); return 0; }); @@ -190,8 +190,8 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 } u32 num = 0; for (u32 i = 0; i < iovcnt; i++) { - u32 ptr = i32_load(iov + i * 8); - u32 len = i32_load(iov + i * 8 + 4); + u32 ptr = wasm_i32_load(iov + i * 8); + u32 len = wasm_i32_load(iov + i * 8 + 4); VERBOSE_LOG(" chunk %d %d\n", ptr, len); ssize_t result = read(nfd, MEMACCESS(ptr), len); if (result < 0) { @@ -204,7 +204,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 } } VERBOSE_LOG(" success: %d\n", num); - i32_store(pnum, num); + wasm_i32_store(pnum, num); return 0; }); @@ -221,8 +221,8 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_closeZ_ii, (u32 fd), { IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_sizes_getZ_iii, (u32 pcount, u32 pbuf_size), { // TODO: connect to actual env? - i32_store(pcount, 0); - i32_store(pbuf_size, 0); + wasm_i32_store(pcount, 0); + wasm_i32_store(pbuf_size, 0); return 0; }); @@ -251,7 +251,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); return WASI_DEFAULT_ERROR; } - i64_store(new_offset, off); + wasm_i64_store(new_offset, off); return 0; }); IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), { @@ -295,25 +295,25 @@ static u32 do_stat(int nfd, u32 buf) { return EM_EACCES; } VERBOSE_LOG(" success, size=%ld\n", nbuf.st_size); - i32_store(buf + 0, nbuf.st_dev); - i32_store(buf + 4, 0); - i32_store(buf + 8, nbuf.st_ino); - i32_store(buf + 12, nbuf.st_mode); - i32_store(buf + 16, nbuf.st_nlink); - i32_store(buf + 20, nbuf.st_uid); - i32_store(buf + 24, nbuf.st_gid); - i32_store(buf + 28, nbuf.st_rdev); - i32_store(buf + 32, 0); - i64_store(buf + 40, nbuf.st_size); - i32_store(buf + 48, nbuf.st_blksize); - i32_store(buf + 52, nbuf.st_blocks); - i32_store(buf + 56, nbuf.st_atim.tv_sec); - i32_store(buf + 60, nbuf.st_atim.tv_nsec); - i32_store(buf + 64, nbuf.st_mtim.tv_sec); - i32_store(buf + 68, nbuf.st_mtim.tv_nsec); - i32_store(buf + 72, nbuf.st_ctim.tv_sec); - i32_store(buf + 76, nbuf.st_ctim.tv_nsec); - i64_store(buf + 80, nbuf.st_ino); + wasm_i32_store(buf + 0, nbuf.st_dev); + wasm_i32_store(buf + 4, 0); + wasm_i32_store(buf + 8, nbuf.st_ino); + wasm_i32_store(buf + 12, nbuf.st_mode); + wasm_i32_store(buf + 16, nbuf.st_nlink); + wasm_i32_store(buf + 20, nbuf.st_uid); + wasm_i32_store(buf + 24, nbuf.st_gid); + wasm_i32_store(buf + 28, nbuf.st_rdev); + wasm_i32_store(buf + 32, 0); + wasm_i64_store(buf + 40, nbuf.st_size); + wasm_i32_store(buf + 48, nbuf.st_blksize); + wasm_i32_store(buf + 52, nbuf.st_blocks); + wasm_i32_store(buf + 56, nbuf.st_atim.tv_sec); + wasm_i32_store(buf + 60, nbuf.st_atim.tv_nsec); + wasm_i32_store(buf + 64, nbuf.st_mtim.tv_sec); + wasm_i32_store(buf + 68, nbuf.st_mtim.tv_nsec); + wasm_i32_store(buf + 72, nbuf.st_ctim.tv_sec); + wasm_i32_store(buf + 76, nbuf.st_ctim.tv_nsec); + wasm_i64_store(buf + 80, nbuf.st_ino); return 0; } @@ -372,12 +372,12 @@ static int main_argc; static char** main_argv; IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_args_sizes_getZ_iii, (u32 pargc, u32 pargv_buf_size), { - i32_store(pargc, main_argc); + wasm_i32_store(pargc, main_argc); u32 buf_size = 0; for (u32 i = 0; i < main_argc; i++) { buf_size += strlen(main_argv[i]) + 1; } - i32_store(pargv_buf_size, buf_size); + wasm_i32_store(pargv_buf_size, buf_size); return 0; }); @@ -385,7 +385,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_args_getZ_iii, (u32 argv, u32 argv_bu u32 buf_size = 0; for (u32 i = 0; i < main_argc; i++) { u32 ptr = argv_buf + buf_size; - i32_store(argv + i * 4, ptr); + wasm_i32_store(argv + i * 4, ptr); char* arg = main_argv[i]; strcpy(MEMACCESS(ptr), arg); buf_size += strlen(arg) + 1; @@ -478,7 +478,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, // wasi expects a result in nanoseconds, and we know how to convert clock() // to seconds, so compute from there const double NSEC_PER_SEC = 1000.0 * 1000.0 * 1000.0; - i64_store(out, (u64)(clock() / (CLOCKS_PER_SEC / NSEC_PER_SEC))); + wasm_i64_store(out, (u64)(clock() / (CLOCKS_PER_SEC / NSEC_PER_SEC))); return 0; }); From 38bc2b5e5573df6946ac91b0522cae954559bf80 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 20 May 2020 20:36:30 -0700 Subject: [PATCH 06/49] working [ci skip] --- tools/shared.py | 7 +++++++ tools/wasm2c/main.c | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index c895e6cf58c6e..1c4f3bfb4a566 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2784,6 +2784,7 @@ def run_wasm_opt(*args, **kwargs): @staticmethod def do_wasm2c(infile): + assert Settings.STANDALONE_WASM # look for the wasm2c/ dir alongside the bin dir, or perhaps higher up. WASM2C_DIR = os.path.dirname(Settings.WABT_BIN) while WASM2C_DIR and not os.path.exists(os.path.join(WASM2C_DIR, 'wasm2c')): @@ -2817,6 +2818,12 @@ def do_wasm2c(infile): total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) with open(c_file, 'w') as out: out.write(total) + ''' in hello world, why this? + }function _emscripten_resize_heap(requestedSize) { + requestedSize = requestedSize >>> 0; + abortOnCannotGrowMemory(requestedSize); + }''' + save_intermediate_counter = 0 diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 1bbd0525b4451..4fc30c62de6c9 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -26,11 +26,13 @@ #define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0) -#define MEMACCESS(addr) ((void*)&Z_envZ_memory->data[addr]) +#define MEMACCESS(addr) ((void*)&Z_memory->data[addr]) +#undef MEMCHECK #define MEMCHECK(a, t) \ - if (UNLIKELY((a) + sizeof(t) > Z_envZ_memory->size)) TRAP(OOB) + if (UNLIKELY((a) + sizeof(t) > Z_memory->size)) TRAP(OOB) +#undef DEFINE_LOAD #define DEFINE_LOAD(name, t1, t2, t3) \ static inline t3 name(u64 addr) { \ MEMCHECK(addr, t1); \ @@ -39,6 +41,7 @@ return (t3)(t2)result; \ } +#undef DEFINE_STORE #define DEFINE_STORE(name, t1, t2) \ static inline void name(u64 addr, t2 value) { \ MEMCHECK(addr, t1); \ From 8114412f7f65a78631568700f0f6aacf2f58e076 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 08:15:06 -0700 Subject: [PATCH 07/49] test passes [ci skip] --- tests/runner.py | 7 ++++++- tests/test_core.py | 21 ++++++++++++++++++++- tools/jsrun.py | 9 +++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/runner.py b/tests/runner.py index a391222c72991..1b54085829103 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -59,7 +59,10 @@ sys.path.append(__rootpath__) import parallel_testsuite -from tools.shared import EM_CONFIG, TEMP_DIR, EMCC, EMXX, DEBUG, PYTHON, LLVM_TARGET, ASM_JS_TARGET, EMSCRIPTEN_TEMP_DIR, WASM_TARGET, SPIDERMONKEY_ENGINE, WINDOWS, EM_BUILD_VERBOSE +from tools.shared import EM_CONFIG, TEMP_DIR, EMCC, EMXX, DEBUG, PYTHON +from tools.shared import LLVM_TARGET, ASM_JS_TARGET, EMSCRIPTEN_TEMP_DIR +from tools.shared import WASM_TARGET, SPIDERMONKEY_ENGINE, WINDOWS +from tools.shared import EM_BUILD_VERBOSE, CLANG_CC from tools.shared import asstr, get_canonical_temp_dir, Building, run_process, try_delete, asbytes, safe_copy, Settings from tools import jsrun, shared, line_endings @@ -1232,6 +1235,8 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, if len(wasm_engines) == 0: logger.warning('no wasm engine was found to run the standalone part of this test') js_engines += wasm_engines + if self.get_setting('WASM2C'): + js_engines += [CLANG_CC] if len(js_engines) == 0: self.skipTest('No JS engine present to run this test with. Check %s and the paths therein.' % EM_CONFIG) for engine in js_engines: diff --git a/tests/test_core.py b/tests/test_core.py index d804f7676aa27..7bb2e50eb0953 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -173,6 +173,25 @@ def decorated(self): return decorated +# Also test wasm2c +def also_with_standalone_wasm_and_wasm2c(func): + def decorated(self): + func(self) + # Standalone mode is only supported in the wasm backend, and not in all + # modes there. + if can_do_standalone(self): + print('standalone') + self.set_setting('STANDALONE_WASM', 1) + func(self) + print('standalone-wasm2c') + self.set_setting('STANDALONE_WASM', 1) + self.set_setting('WASM2C', 1) + self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') + func(self) + + return decorated + + # Similar to also_with_standalone_wasm, but suitable for tests that cannot # run in a wasm VM yet, as they are not 100% standalone. We can still # run them with the JS code though. @@ -353,7 +372,7 @@ def get_bullet_library(self, use_cmake): configure_args=configure_args, cache_name_extra=configure_commands[0]) - @also_with_standalone_wasm + @also_with_standalone_wasm_and_wasm2c def test_hello_world(self): self.do_run_in_out_file_test('tests', 'core', 'test_hello_world') diff --git a/tools/jsrun.py b/tools/jsrun.py index e8c197e08355b..790021df9baec 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -47,6 +47,7 @@ def make_command(filename, engine=None, args=[]): is_jsc = 'jsc' in jsengine is_wasmer = 'wasmer' in jsengine is_wasmtime = 'wasmtime' in jsengine + is_clang = 'clang' in jsengine # Disable true async compilation (async apis will in fact be synchronous) for now # due to https://bugs.chromium.org/p/v8/issues/detail?id=6263 shell_option_flags = ['--no-wasm-async-compilation'] if is_d8 else [] @@ -56,6 +57,14 @@ def make_command(filename, engine=None, args=[]): if is_wasmer or is_wasmtime: # in a wasm runtime, run the wasm, not the js filename = shared.unsuffixed(filename) + '.wasm' + elif is_clang: + # with wasm2c, the input is a c file, which we must compile first + c = shared.unsuffixed(filename) + '.c' + executable = shared.unsuffixed(filename) + '.exe' + shared.run_process(engine + [c, '-o', executable]) + # we can now run the executable directly, without an engine + engine = [] + filename = executable # Separates engine flags from script flags flag_separator = ['--'] if is_d8 or is_jsc else [] return engine + command_flags + [filename] + shell_option_flags + flag_separator + args From 8bdf87ee1b407d9cc697984ef3e940ab45e590a5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 08:19:52 -0700 Subject: [PATCH 08/49] another test --- tests/test_core.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 7bb2e50eb0953..89df4f7a73ba1 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -173,7 +173,6 @@ def decorated(self): return decorated -# Also test wasm2c def also_with_standalone_wasm_and_wasm2c(func): def decorated(self): func(self) @@ -183,7 +182,7 @@ def decorated(self): print('standalone') self.set_setting('STANDALONE_WASM', 1) func(self) - print('standalone-wasm2c') + print('wasm2c') self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') @@ -214,6 +213,30 @@ def decorated(self): return decorated +def also_with_impure_standalone_wasm_and_wasm2c(func): + def decorated(self): + func(self) + # Standalone mode is only supported in the wasm backend, and not in all + # modes there. + if can_do_standalone(self): + print('standalone (impure; no wasm runtimes)') + with wasm_engines_modify([]): + self.set_setting('STANDALONE_WASM', 1) + # we will not legalize the JS ffi interface, so we must use BigInt + # support in order for JS to have a chance to run this without trapping + # when it sees an i64 on the ffi. + self.set_setting('WASM_BIGINT', 1) + with js_engines_modify([NODE_JS + ['--experimental-wasm-bigint']]): + func(self) + print('wasm2c') + self.set_setting('STANDALONE_WASM', 1) + self.set_setting('WASM2C', 1) + self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') + func(self) + + return decorated + + # Similar to also_with_standalone_wasm, but suitable for tests that can *only* # run in a wasm VM, or in non-standalone mode, but not in standalone mode with # our JS. @@ -1099,6 +1122,7 @@ def test_wcslen(self): def test_regex(self): self.do_run_in_out_file_test('tests', 'core', 'test_regex') + @also_with_impure_standalone_wasm_and_wasm2c def test_longjmp(self): self.do_run_in_out_file_test('tests', 'core', 'test_longjmp') From cfbc317eda8a65886fa639f61fbb7d8222dcae9e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 08:26:17 -0700 Subject: [PATCH 09/49] another [ci skip] --- tests/test_core.py | 6 ++++-- tools/wasm2c/main.c | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 89df4f7a73ba1..17b2237bbdf70 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -232,7 +232,9 @@ def decorated(self): self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') - func(self) + # disable js engines too, so we only run the c output + with js_engines_modify([]): + func(self) return decorated @@ -6621,7 +6623,7 @@ def do_autodebug(filename): @no_asan('autodebug logging interferes with asan') @no_fastcomp('autodebugging wasm is only supported in the wasm backend') @with_env_modify({'EMCC_AUTODEBUG': '1'}) - @also_with_impure_standalone_wasm + @also_with_impure_standalone_wasm_and_wasm2c def test_autodebug_wasm(self): # Autodebug does not work with too much shadow memory. # Memory consumed by autodebug depends on the size of the WASM linear memory. diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 4fc30c62de6c9..052a1855f443a 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -554,6 +554,10 @@ IMPORT_IMPL(u32, Z_envZ_load_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { tempRet0 = high; return low; }); +IMPORT_IMPL(u64, Z_envZ_load_val_i64Z_jij, (u32 loc, u64 value), { + printf("load_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); + return value; +}); IMPORT_IMPL(f32, Z_envZ_load_val_f32Z_fif, (u32 loc, f32 value), { printf("load_val_f32 %d,%f\n", loc, value); return value; @@ -575,6 +579,10 @@ IMPORT_IMPL(u32, Z_envZ_store_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { tempRet0 = high; return low; }); +IMPORT_IMPL(u64, Z_envZ_store_val_i64Z_jij, (u32 loc, u64 value), { + printf("store_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); + return value; +}); IMPORT_IMPL(f32, Z_envZ_store_val_f32Z_fif, (u32 loc, f32 value), { printf("store_val_f32 %d,%f\n", loc, value); return value; From e8ad395e8d0c1bbfda4f8e9feaa2ed9b54c16897 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 08:35:54 -0700 Subject: [PATCH 10/49] another test [ci skip] --- tests/test_core.py | 24 ++++++++++++++++++++++-- tools/wasm2c/main.c | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 17b2237bbdf70..5b5e0d7f16e20 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -186,7 +186,8 @@ def decorated(self): self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') - func(self) + with wasm_engines_modify([]): + func(self) return decorated @@ -255,6 +256,25 @@ def decorated(self): return decorated +def also_with_only_standalone_wasm_and_wasm2c(func): + def decorated(self): + func(self) + # Standalone mode is only supported in the wasm backend, and not in all + # modes there. + if can_do_standalone(self): + with js_engines_modify([]): + print('standalone (only; no js runtimes)') + self.set_setting('STANDALONE_WASM', 1) + func(self) + print('wasm2c') + self.set_setting('STANDALONE_WASM', 1) + self.set_setting('WASM2C', 1) + self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') + with wasm_engines_modify([]): + func(self) + return decorated + + def node_pthreads(f): def decorated(self): self.set_setting('USE_PTHREADS', 1) @@ -5637,7 +5657,7 @@ def test_unistd_misc(self): # i64s in the API, which we'd need to legalize for JS, so in standalone mode # all we can test is wasm VMs - @also_with_only_standalone_wasm + @also_with_only_standalone_wasm_and_wasm2c def test_posixtime(self): test_path = path_from_root('tests', 'core', 'test_posixtime') src, output = (test_path + s for s in ('.c', '.out')) diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 052a1855f443a..264e1ab1949a5 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -91,6 +91,7 @@ ret (*name) params = _##name; #define STUB_IMPORT_IMPL(ret, name, params, returncode) IMPORT_IMPL(ret, name, params, { return returncode; }); #define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ +#define WASI_EINVAL 28 IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { exit(x); @@ -476,7 +477,13 @@ IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { longjmp(setjmp_stack[next_setjmp - 1], 1); }); +#define WASM_CLOCK_REALTIME 0 +#define WASM_CLOCK_MONOTONIC 1 + IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { + if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC) { + return WASI_EINVAL; + } // TODO: handle realtime vs monotonic etc. // wasi expects a result in nanoseconds, and we know how to convert clock() // to seconds, so compute from there @@ -485,6 +492,15 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, return 0; }); +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), { + if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC) { + return WASI_EINVAL; + } + // TODO: handle realtime vs monotonic etc. For now just report "milliseconds". + wasm_i64_store(out, 1000 * 1000); + return 0; +}); + IMPORT_IMPL(void, Z_envZ_emscripten_notify_memory_growthZ_vi, (u32 size), {}); STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); From efd17eed4d816c6632371cf9f53c17f8ee786ab0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 11:11:26 -0700 Subject: [PATCH 11/49] more --- package-lock.json | 5 +++++ package.json | 3 ++- src/settings.js | 4 ---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index b3819c01bf32a..9815f76c67633 100644 --- a/package-lock.json +++ b/package-lock.json @@ -361,6 +361,11 @@ "source-map": "^0.5.1" } }, + "wabt": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/wabt/-/wabt-1.0.16.tgz", + "integrity": "sha512-aSQEAJfYkoZRY9Qt2/6hTM8yRX/Nc2R1bScET/4lppRqfUyzynB5HI+lK0u/hp8NbCVTAXg0iETviSS3zoufJw==" + }, "ws": { "version": "0.4.32", "resolved": "http://registry.npmjs.org/ws/-/ws-0.4.32.tgz", diff --git a/package.json b/package.json index 4bcb89aaf39e3..9a70092fd2ab9 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "dependencies": { "google-closure-compiler": "20200224.0.0", "html-minifier-terser": "5.0.2", - "source-map": "0.5.6" + "source-map": "0.5.6", + "wabt": "^1.0.16" } } diff --git a/src/settings.js b/src/settings.js index 8f263f04264db..f239f1cad4465 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1756,11 +1756,7 @@ var PRINTF_LONG_DOUBLE = 0; // Run wabt's wasm2c tool on the final wasm, and combine that with a C runtime, // resulting in a .c file that you can compile with a C compiler to get a // native executable that works the same as the normal js+wasm. -// When using this you must specify WABT_BIN, which should be where the wasm2c -// executable can be found. We will also look for the wasm2c directory near it -// as we need headers and other support there. var WASM2C = 0; -var WABT_BIN = ''; //=========================================== // Internal, used for testing only, from here From 5d8fc9696b8879d2e199fe8356adcc8c2d3f88e7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 11:18:44 -0700 Subject: [PATCH 12/49] works with js [ci skip] --- tests/test_core.py | 3 --- tools/shared.py | 10 +++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 5b5e0d7f16e20..fdc31619250ea 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -185,7 +185,6 @@ def decorated(self): print('wasm2c') self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) - self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') with wasm_engines_modify([]): func(self) @@ -232,7 +231,6 @@ def decorated(self): print('wasm2c') self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) - self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') # disable js engines too, so we only run the c output with js_engines_modify([]): func(self) @@ -269,7 +267,6 @@ def decorated(self): print('wasm2c') self.set_setting('STANDALONE_WASM', 1) self.set_setting('WASM2C', 1) - self.set_setting('WABT_BIN', '/home/azakai/Dev/wabt/build') with wasm_engines_modify([]): func(self) return decorated diff --git a/tools/shared.py b/tools/shared.py index 1c4f3bfb4a566..24f1b1b21142b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2786,15 +2786,11 @@ def run_wasm_opt(*args, **kwargs): def do_wasm2c(infile): assert Settings.STANDALONE_WASM # look for the wasm2c/ dir alongside the bin dir, or perhaps higher up. - WASM2C_DIR = os.path.dirname(Settings.WABT_BIN) - while WASM2C_DIR and not os.path.exists(os.path.join(WASM2C_DIR, 'wasm2c')): - WASM2C_DIR = os.path.dirname(WASM2C_DIR) - if not WASM2C_DIR: - exit_with_error('Could not find wabt wasm2c/ dir in the tree under ' + Settings.WABT_BIN) - WASM2C_DIR = os.path.join(WASM2C_DIR, 'wasm2c') + WASM2C = NODE_JS + [path_from_root('node_modules', 'wabt', 'wasm2c.js')] + WASM2C_DIR = path_from_root('node_modules', 'wabt', 'wasm2c') c_file = unsuffixed(infile) + '.c' h_file = unsuffixed(infile) + '.h' - cmd = [os.path.join(Settings.WABT_BIN, 'wasm2c'), infile, '-o', c_file] + cmd = WASM2C + [infile, '-o', c_file] run_process(cmd) with open(c_file) as read_c: c = read_c.read() From e555a0473b1b74839da818d48adfdc0c02867471 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 11:54:03 -0700 Subject: [PATCH 13/49] fixes [ci skip] --- tools/wasm2c/main.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 264e1ab1949a5..1ff2fa284acd1 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -416,6 +416,14 @@ ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; DECLARE_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); +// Stack support should be linked in if it is needed. +IMPORT_IMPL(__attribute__((weak)) u32, Z_stackSaveZ_iv, (), { + abort(); +}); +IMPORT_IMPL(__attribute__((weak))void, Z_stackRestoreZ_vi, (u32 x), { + abort(); +}); + #define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ DECLARE_EXPORT(void, dyncall, types); \ \ From 6fcb454acd22f9d417490d9bc971f58db81ca78f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 12:09:11 -0700 Subject: [PATCH 14/49] benches [ci skip] --- tests/test_benchmark.py | 21 ++++++++++----------- tools/shared.py | 7 +++++++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 21ea338ccfe32..3d7f53018cb01 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -8,6 +8,7 @@ import os import re import shutil +import subprocess import sys import time import unittest @@ -246,22 +247,19 @@ def __init__(self, name): def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): # wasm2c doesn't want minimal runtime which the normal emscripten # benchmarker defaults to, as we don't have any JS anyhow - emcc_args = emcc_args + ['-s', 'STANDALONE_WASM', '-s', 'MINIMAL_RUNTIME=0'] # WASM2C flag? + emcc_args = emcc_args + [ + '-s', 'STANDALONE_WASM', + '-s', 'MINIMAL_RUNTIME=0', + '-s', 'WASM2C' + ] super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) base = self.filename[:-3] - wasm = base + '.wasm' c = base + '.c' - h = base + '.h' native = base + '.exe' - wabt_dir = os.path.expanduser('~/Dev/wabt/') - run_process([os.path.join(wabt_dir, 'build', 'wasm2c'), wasm, '-o', c]) - run_process(['clang', os.path.join(wabt_dir, 'wasm2c', 'main-emscripten.c'), c, - os.path.join(wabt_dir, 'wasm2c', 'wasm-rt-impl.c'), '-I.', - '-I' + os.path.join(wabt_dir, 'wasm2c'), '-lm', - '-include', h, '-o', native, OPTIMIZATIONS, + run_process(['clang', c, '-o', native, OPTIMIZATIONS, '-lm', '-DWASM_RT_MAX_CALL_STACK_DEPTH=8000']) # for havlak self.filename = native @@ -356,8 +354,9 @@ def cleanup(self): aot_v8 = V8_ENGINE + ['--no-liftoff'] default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8' benchmarkers += [ - EmscriptenBenchmarker(default_v8_name, aot_v8), - EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + #EmscriptenBenchmarker(default_v8_name, aot_v8), + #EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + EmscriptenWasm2CBenchmarker('wasm2c') ] if os.path.exists(CHEERP_BIN): benchmarkers += [ diff --git a/tools/shared.py b/tools/shared.py index 24f1b1b21142b..e9dc80a5e3984 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2792,6 +2792,13 @@ def do_wasm2c(infile): h_file = unsuffixed(infile) + '.h' cmd = WASM2C + [infile, '-o', c_file] run_process(cmd) + + ''' + // comment with + run_process(['clang', c, '-o', native, OPTIMIZATIONS, '-lm', + '-DWASM_RT_MAX_CALL_STACK_DEPTH=8000']) # for havlak + ''' + with open(c_file) as read_c: c = read_c.read() SEP = '\n//====================\n\n' From fa81d58591ade2ba9092ab802b273799e84a196c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:00:39 -0700 Subject: [PATCH 15/49] fix [ci skip] --- tools/wasm2c/main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 1ff2fa284acd1..76a5292cefcb8 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -487,9 +487,10 @@ IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { #define WASM_CLOCK_REALTIME 0 #define WASM_CLOCK_MONOTONIC 1 +#define WASM_CLOCK_PROCESS_CPUTIME 2 IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { - if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC) { + if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC && clock_id != WASM_CLOCK_PROCESS_CPUTIME) { return WASI_EINVAL; } // TODO: handle realtime vs monotonic etc. @@ -501,7 +502,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, }); IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), { - if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC) { + if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC && clock_id != WASM_CLOCK_PROCESS_CPUTIME) { return WASI_EINVAL; } // TODO: handle realtime vs monotonic etc. For now just report "milliseconds". From 77615913a1a2264937bc41ba5200ceb880fa9707 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:02:51 -0700 Subject: [PATCH 16/49] fix [ci skip] --- tools/wasm2c/main.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 76a5292cefcb8..02abb602d971e 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -488,9 +488,15 @@ IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { #define WASM_CLOCK_REALTIME 0 #define WASM_CLOCK_MONOTONIC 1 #define WASM_CLOCK_PROCESS_CPUTIME 2 +#define WASM_CLOCK_CLOCK_THREAD_CPUTIME_ID 3 + +static int check_clock(u32 clock_id) { + return clock_id == WASM_CLOCK_REALTIME || clock_id == WASM_CLOCK_MONOTONIC || + clock_id == WASM_CLOCK_PROCESS_CPUTIME || clock_id == CLOCK_THREAD_CPUTIME_ID; +} IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { - if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC && clock_id != WASM_CLOCK_PROCESS_CPUTIME) { + if (!check_clock(clock_id)) { return WASI_EINVAL; } // TODO: handle realtime vs monotonic etc. @@ -502,7 +508,7 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, }); IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), { - if (clock_id != WASM_CLOCK_REALTIME && clock_id != WASM_CLOCK_MONOTONIC && clock_id != WASM_CLOCK_PROCESS_CPUTIME) { + if (!check_clock(clock_id)) { return WASI_EINVAL; } // TODO: handle realtime vs monotonic etc. For now just report "milliseconds". From f04b0e3912e517f95dff99c9d77d82de10f58283 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:23:05 -0700 Subject: [PATCH 17/49] temp [ci skip] --- tests/test_benchmark.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 3d7f53018cb01..337992497b829 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -45,9 +45,9 @@ OPTIMIZATIONS = '-O3' -PROFILING = 0 +PROFILING = 1 -LLVM_FEATURE_FLAGS = ['-mnontrapping-fptoint'] +LLVM_FEATURE_FLAGS = [] # ['-mnontrapping-fptoint'] class Benchmarker(object): @@ -213,7 +213,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat if 'FORCE_FILESYSTEM=1' in cmd: cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd] if PROFILING: - cmd += ['--profiling-funcs'] + cmd += ['--profiling']#-funcs'] self.cmd = cmd run_process(cmd, env=self.env) if self.binaryen_opts: @@ -255,6 +255,9 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) + # move the JS away so there is no chance we run it by mistake + shutil.move(self.filename, self.filename + '.old.js') + base = self.filename[:-3] c = base + '.c' native = base + '.exe' From 9b7f095b83b0753860676305d86989987a74772b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:40:03 -0700 Subject: [PATCH 18/49] fixes [ci skip] --- tests/test_core.py | 2 +- tools/jsrun.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index fdc31619250ea..fc84b0d16f3c4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -586,7 +586,7 @@ def test_cube2md5(self): shutil.copyfile(path_from_root('tests', 'cube2md5.txt'), 'cube2md5.txt') self.do_run(open(path_from_root('tests', 'cube2md5.cpp')).read(), open(path_from_root('tests', 'cube2md5.ok')).read(), assert_returncode=None) - @also_with_standalone_wasm + @also_with_standalone_wasm_and_wasm2c @needs_make('make') def test_cube2hash(self): # A good test of i64 math diff --git a/tools/jsrun.py b/tools/jsrun.py index 790021df9baec..2a40bb12b711e 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -64,7 +64,7 @@ def make_command(filename, engine=None, args=[]): shared.run_process(engine + [c, '-o', executable]) # we can now run the executable directly, without an engine engine = [] - filename = executable + filename = os.path.abspath(executable) # Separates engine flags from script flags flag_separator = ['--'] if is_d8 or is_jsc else [] return engine + command_flags + [filename] + shell_option_flags + flag_separator + args @@ -92,10 +92,14 @@ def check_engine(engine): def require_engine(engine): engine_path = engine[0] + # an empty engine means we are running an executable directly somehow; there + # is nothing to check here + if engine_path == '/': + return if engine_path not in WORKING_ENGINES: check_engine(engine) if not WORKING_ENGINES[engine_path]: - logging.critical('The JavaScript shell (%s) does not seem to work, check the paths in the config file' % engine) + logging.critical('The engine (%s) does not seem to work, check the paths in the config file' % engine) sys.exit(1) From eab98092809c66509140b2689a9567307d6ca5a0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:49:03 -0700 Subject: [PATCH 19/49] fix --- emcc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/emcc.py b/emcc.py index 96253585b5f3a..3aedb6cc150d0 100755 --- a/emcc.py +++ b/emcc.py @@ -3339,8 +3339,6 @@ def run_closure_compiler(final): shared.Building.emit_debug_on_side(wasm_binary_target, dwarf_target) if shared.Settings.WASM2C: - # FIXME this assumes the emsdk layout where the binaryen and wabt binaries - # are together shared.Building.do_wasm2c(wasm_binary_target) # replace placeholder strings with correct subresource locations From 3639b365de1332d49b5b90dd249ba3e90ffa3f50 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 13:54:11 -0700 Subject: [PATCH 20/49] cleanup [ci skip] --- tools/shared.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index e9dc80a5e3984..92198633c73e8 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2785,48 +2785,43 @@ def run_wasm_opt(*args, **kwargs): @staticmethod def do_wasm2c(infile): assert Settings.STANDALONE_WASM - # look for the wasm2c/ dir alongside the bin dir, or perhaps higher up. WASM2C = NODE_JS + [path_from_root('node_modules', 'wabt', 'wasm2c.js')] WASM2C_DIR = path_from_root('node_modules', 'wabt', 'wasm2c') c_file = unsuffixed(infile) + '.c' h_file = unsuffixed(infile) + '.h' cmd = WASM2C + [infile, '-o', c_file] run_process(cmd) - - ''' - // comment with - run_process(['clang', c, '-o', native, OPTIMIZATIONS, '-lm', - '-DWASM_RT_MAX_CALL_STACK_DEPTH=8000']) # for havlak - ''' - - with open(c_file) as read_c: - c = read_c.read() - SEP = '\n//====================\n\n' + total = '''\ +/* + * This file was generated by emcc+wasm2c. To compile it, use something like + * + * $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 + */ +''' + SEP = '\n/* ==================================== */\n' # hermeticize the C file, by bundling in the wasm2c/ includes headers = [ (WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'), (os.path.dirname(h_file), os.path.basename(h_file)) ] - total = '' for header in headers: with open(os.path.join(header[0], header[1])) as f: total += f.read() + SEP - total += c + SEP + # add the wasm2c output + with open(c_file) as read_c: + total += read_c.read() + SEP + # add the wasm2c runtime with open(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) as f: total += f.read() + SEP + # add the emscripten main with open(path_from_root('tools', 'wasm2c', 'main.c')) as main: total += main.read() + # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) with open(c_file, 'w') as out: out.write(total) - ''' in hello world, why this? - }function _emscripten_resize_heap(requestedSize) { - requestedSize = requestedSize >>> 0; - abortOnCannotGrowMemory(requestedSize); - }''' - save_intermediate_counter = 0 From dd0bdb30b0d39afd14597dbe7754b7eed8fee5eb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 15:06:38 -0700 Subject: [PATCH 21/49] autogen invokes [ci skip] --- tools/shared.py | 41 ++++++++++++++++++++++++++++++++++++++++- tools/wasm2c/main.c | 7 +------ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 92198633c73e8..5ec408ecfd2ad 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2810,7 +2810,8 @@ def do_wasm2c(infile): total += f.read() + SEP # add the wasm2c output with open(c_file) as read_c: - total += read_c.read() + SEP + c = read_c.read() + total += c + SEP # add the wasm2c runtime with open(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) as f: total += f.read() + SEP @@ -2820,6 +2821,44 @@ def do_wasm2c(infile): # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) + # generate the necessary invokes + invokes = [] + for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): + def s_to_c(s): + if s == 'v': + return 'void' + elif s == 'i': + return 'u32' + elif s == 'j': + return 'u64' + elif s == 'f': + return 'f32' + elif s == 'd': + return 'f64' + else: + exit_with_error('invalid sig element:' + str(s)) + + def name(i): + return 'a' + str(i) + + wabt_sig = sig[0] + 'i' + sig[1:] + typed_args = ['u32 fptr'] + [s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))] + types = ['u32'] + [s_to_c(sig[i]) for i in range(1, len(sig))] + args = ['fptr'] + [name(i) for i in range(1, len(sig))] + invokes.append( + '%s_INVOKE_IMPL(%sZ_envZ_invoke_%sZ_%s, (%s), (%s), (%s), Z_dynCall_%sZ_%s);' % ( + 'VOID' if sig[0] == 'v' else 'RETURNING', + (s_to_c(sig[0]) + ', ') if sig[0] != 'v' else '', + sig, + wabt_sig, + ', '.join(typed_args), + ', '.join(types), + ', '.join(args), + sig, + wabt_sig + )) + total = total.replace('/* {{{ EMCC_INVOKE_IMPLS }}} */', '\n'.join(invokes)) + # write out the final file with open(c_file, 'w') as out: out.write(total) diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 02abb602d971e..3ff1983ac3094 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -470,12 +470,7 @@ IMPORT_IMPL(ret, name, typed_args, { \ return returned_value; \ }); -VOID_INVOKE_IMPL(Z_envZ_invoke_vZ_vi, (u32 fptr), (u32), (fptr), Z_dynCall_vZ_vi); -VOID_INVOKE_IMPL(Z_envZ_invoke_viiZ_viii, (u32 fptr, u32 a, u32 b), (u32, u32, u32), (fptr, a, b), Z_dynCall_viiZ_viii); -VOID_INVOKE_IMPL(Z_envZ_invoke_viiiZ_viiii, (u32 fptr, u32 a, u32 b, u32 c), (u32, u32, u32, u32), (fptr, a, b, c), Z_dynCall_viiiZ_viiii); - -RETURNING_INVOKE_IMPL(u32, Z_envZ_invoke_iiiZ_iiii, (u32 fptr, u32 a, u32 b), (u32, u32, u32), (fptr, a, b), Z_dynCall_iiiZ_iiii); -RETURNING_INVOKE_IMPL(u32, Z_envZ_invoke_iiZ_iii, (u32 fptr, u32 a), (u32, u32), (fptr, a), Z_dynCall_iiZ_iii); +/* {{{ EMCC_INVOKE_IMPLS }}} */ IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { if (next_setjmp == 0) { From 20eda6f637840ac87a28342ab6873aa8f716a4d0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 15:33:35 -0700 Subject: [PATCH 22/49] npm --- package-lock.json | 8 ++++---- package.json | 2 +- tools/shared.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9815f76c67633..b87b1e947af9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -361,10 +361,10 @@ "source-map": "^0.5.1" } }, - "wabt": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/wabt/-/wabt-1.0.16.tgz", - "integrity": "sha512-aSQEAJfYkoZRY9Qt2/6hTM8yRX/Nc2R1bScET/4lppRqfUyzynB5HI+lK0u/hp8NbCVTAXg0iETviSS3zoufJw==" + "wasm2c": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wasm2c/-/wasm2c-1.0.0.tgz", + "integrity": "sha512-4SIESF2JNxrry6XFa/UQcsQibn+bxPkQ/oqixiXz2o8fsMl8J4vtvhH/evgbi8vZajAlaukuihEcQTWb9tVLUA==" }, "ws": { "version": "0.4.32", diff --git a/package.json b/package.json index 9a70092fd2ab9..17de4cc07972a 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,6 @@ "google-closure-compiler": "20200224.0.0", "html-minifier-terser": "5.0.2", "source-map": "0.5.6", - "wabt": "^1.0.16" + "wasm2c": "1.0.0" } } diff --git a/tools/shared.py b/tools/shared.py index 5ec408ecfd2ad..1df2449b23d3b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2785,8 +2785,8 @@ def run_wasm_opt(*args, **kwargs): @staticmethod def do_wasm2c(infile): assert Settings.STANDALONE_WASM - WASM2C = NODE_JS + [path_from_root('node_modules', 'wabt', 'wasm2c.js')] - WASM2C_DIR = path_from_root('node_modules', 'wabt', 'wasm2c') + WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] + WASM2C_DIR = path_from_root('node_modules', 'wasm2c') c_file = unsuffixed(infile) + '.c' h_file = unsuffixed(infile) + '.h' cmd = WASM2C + [infile, '-o', c_file] From 67c68f437ae723d1cbdc10ec6996ed8b1d3fcd16 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 15:34:49 -0700 Subject: [PATCH 23/49] cleanup --- tests/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runner.py b/tests/runner.py index 1b54085829103..06245b7378abc 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -1236,6 +1236,7 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, logger.warning('no wasm engine was found to run the standalone part of this test') js_engines += wasm_engines if self.get_setting('WASM2C'): + # the "engine" to run wasm2c builds is clang that compiles the c js_engines += [CLANG_CC] if len(js_engines) == 0: self.skipTest('No JS engine present to run this test with. Check %s and the paths therein.' % EM_CONFIG) From 3f0d8dc8f80dfb4084b3fe791880c96863e492e7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 15:37:53 -0700 Subject: [PATCH 24/49] more --- tests/test_benchmark.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 337992497b829..256f7b9322d58 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -47,7 +47,7 @@ PROFILING = 1 -LLVM_FEATURE_FLAGS = [] # ['-mnontrapping-fptoint'] +LLVM_FEATURE_FLAGS = ['-mnontrapping-fptoint'] class Benchmarker(object): @@ -213,7 +213,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat if 'FORCE_FILESYSTEM=1' in cmd: cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd] if PROFILING: - cmd += ['--profiling']#-funcs'] + cmd += ['--profiling-funcs'] self.cmd = cmd run_process(cmd, env=self.env) if self.binaryen_opts: @@ -253,7 +253,14 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat '-s', 'WASM2C' ] - super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) + global LLVM_FEATURE_FLAGS + old_flags = LLVM_FEATURE_FLAGS + try: + # wasm2c does not support anything beyond MVP + LLVM_FEATURE_FLAGS = [] + super(EmscriptenWasm2CBenchmarker, self).build(parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser) + finally: + LLVM_FEATURE_FLAGS = old_flags # move the JS away so there is no chance we run it by mistake shutil.move(self.filename, self.filename + '.old.js') @@ -357,9 +364,9 @@ def cleanup(self): aot_v8 = V8_ENGINE + ['--no-liftoff'] default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8' benchmarkers += [ - #EmscriptenBenchmarker(default_v8_name, aot_v8), - #EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), - EmscriptenWasm2CBenchmarker('wasm2c') + EmscriptenBenchmarker(default_v8_name, aot_v8), + EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']), + # EmscriptenWasm2CBenchmarker('wasm2c') ] if os.path.exists(CHEERP_BIN): benchmarkers += [ From 6e5bbfc926c25423c1198a7009ce317399bf1f5d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 16:02:54 -0700 Subject: [PATCH 25/49] better --- tests/test_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index fc84b0d16f3c4..180494c9b27d6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -414,7 +414,6 @@ def get_bullet_library(self, use_cmake): configure_args=configure_args, cache_name_extra=configure_commands[0]) - @also_with_standalone_wasm_and_wasm2c def test_hello_world(self): self.do_run_in_out_file_test('tests', 'core', 'test_hello_world') From c445c0e9a27ee63789f8e079825cecf2cac5321f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 16:37:44 -0700 Subject: [PATCH 26/49] fixes --- tests/runner.py | 2 +- tests/test_benchmark.py | 2 +- tools/jsrun.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/runner.py b/tests/runner.py index 06245b7378abc..2c55f53cc6648 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -1237,7 +1237,7 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, js_engines += wasm_engines if self.get_setting('WASM2C'): # the "engine" to run wasm2c builds is clang that compiles the c - js_engines += [CLANG_CC] + js_engines += [[CLANG_CC]] if len(js_engines) == 0: self.skipTest('No JS engine present to run this test with. Check %s and the paths therein.' % EM_CONFIG) for engine in js_engines: diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 256f7b9322d58..d1c93db40dd15 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -45,7 +45,7 @@ OPTIMIZATIONS = '-O3' -PROFILING = 1 +PROFILING = 0 LLVM_FEATURE_FLAGS = ['-mnontrapping-fptoint'] diff --git a/tools/jsrun.py b/tools/jsrun.py index 2a40bb12b711e..c9f551c14d665 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -92,9 +92,9 @@ def check_engine(engine): def require_engine(engine): engine_path = engine[0] - # an empty engine means we are running an executable directly somehow; there - # is nothing to check here - if engine_path == '/': + # if clang is the "engine", it means we compiled to a native executable; + # there is nothing to check here + if engine_path == shared.CLANG_CC: return if engine_path not in WORKING_ENGINES: check_engine(engine) From f450a40efda0b13840b23ff7d7a569d7bf6fee37 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 16:41:52 -0700 Subject: [PATCH 27/49] nicer --- tools/jsrun.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/jsrun.py b/tools/jsrun.py index c9f551c14d665..d13f20b89bac1 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -48,6 +48,7 @@ def make_command(filename, engine=None, args=[]): is_wasmer = 'wasmer' in jsengine is_wasmtime = 'wasmtime' in jsengine is_clang = 'clang' in jsengine + is_clang = engine[0] == shared.CLANG_CC # Disable true async compilation (async apis will in fact be synchronous) for now # due to https://bugs.chromium.org/p/v8/issues/detail?id=6263 shell_option_flags = ['--no-wasm-async-compilation'] if is_d8 else [] From 649d7dd8739e86db9ae0076026de7f70568557fc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 21 May 2020 16:42:06 -0700 Subject: [PATCH 28/49] nicer --- tools/jsrun.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/jsrun.py b/tools/jsrun.py index d13f20b89bac1..526eb88608f3c 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -47,7 +47,6 @@ def make_command(filename, engine=None, args=[]): is_jsc = 'jsc' in jsengine is_wasmer = 'wasmer' in jsengine is_wasmtime = 'wasmtime' in jsengine - is_clang = 'clang' in jsengine is_clang = engine[0] == shared.CLANG_CC # Disable true async compilation (async apis will in fact be synchronous) for now # due to https://bugs.chromium.org/p/v8/issues/detail?id=6263 From 2e449e5f0b96ee04b114731eb10492714266c971 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 13:32:44 -0700 Subject: [PATCH 29/49] fix suffix [ci skip] --- src/settings.js | 5 ++++- tests/test_benchmark.py | 2 +- tools/jsrun.py | 2 +- tools/shared.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/settings.js b/src/settings.js index f239f1cad4465..09e8770fe1172 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1755,7 +1755,10 @@ var PRINTF_LONG_DOUBLE = 0; // Run wabt's wasm2c tool on the final wasm, and combine that with a C runtime, // resulting in a .c file that you can compile with a C compiler to get a -// native executable that works the same as the normal js+wasm. +// native executable that works the same as the normal js+wasm. This will also +// emit the wasm2c .h file. The output filenames will be X.wasm.c, X.wasm.h +// if your output is X.js or X.wasm (note the added .wasm. we make sure to emit, +// which avoids trampling a C file). var WASM2C = 0; //=========================================== diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index d1c93db40dd15..c5f1006137004 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -266,7 +266,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat shutil.move(self.filename, self.filename + '.old.js') base = self.filename[:-3] - c = base + '.c' + c = base + '.wasm.c' native = base + '.exe' run_process(['clang', c, '-o', native, OPTIMIZATIONS, '-lm', diff --git a/tools/jsrun.py b/tools/jsrun.py index 526eb88608f3c..6da0461fb2a7c 100644 --- a/tools/jsrun.py +++ b/tools/jsrun.py @@ -59,7 +59,7 @@ def make_command(filename, engine=None, args=[]): filename = shared.unsuffixed(filename) + '.wasm' elif is_clang: # with wasm2c, the input is a c file, which we must compile first - c = shared.unsuffixed(filename) + '.c' + c = shared.unsuffixed(filename) + '.wasm.c' executable = shared.unsuffixed(filename) + '.exe' shared.run_process(engine + [c, '-o', executable]) # we can now run the executable directly, without an engine diff --git a/tools/shared.py b/tools/shared.py index 1df2449b23d3b..0730a89bd83a2 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2787,8 +2787,8 @@ def do_wasm2c(infile): assert Settings.STANDALONE_WASM WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] WASM2C_DIR = path_from_root('node_modules', 'wasm2c') - c_file = unsuffixed(infile) + '.c' - h_file = unsuffixed(infile) + '.h' + c_file = unsuffixed(infile) + '.wasm.c' + h_file = unsuffixed(infile) + '.wasm.h' cmd = WASM2C + [infile, '-o', c_file] run_process(cmd) total = '''\ From 30c17546c7ba7679b9e7b6a94dc724034e7b4da9 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 13:43:47 -0700 Subject: [PATCH 30/49] splitting --- tools/shared.py | 14 +- tools/wasm2c/autodebug.c | 87 ++++++ tools/wasm2c/base.c | 183 ++++++++++++ tools/wasm2c/main.c | 597 --------------------------------------- tools/wasm2c/os.c | 316 +++++++++++++++++++++ 5 files changed, 596 insertions(+), 601 deletions(-) create mode 100644 tools/wasm2c/autodebug.c create mode 100644 tools/wasm2c/base.c create mode 100644 tools/wasm2c/os.c diff --git a/tools/shared.py b/tools/shared.py index 0730a89bd83a2..fb872a478b621 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2815,9 +2815,15 @@ def do_wasm2c(infile): # add the wasm2c runtime with open(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) as f: total += f.read() + SEP - # add the emscripten main - with open(path_from_root('tools', 'wasm2c', 'main.c')) as main: - total += main.read() + # add the support code + support_files = ['base'] + if Settings.AUTODEBUG: + support_files.append('autodebug') + if Settings.EXPECT_MAIN: + support_files.append('main') + for support_file in support_files: + with open(path_from_root('tools', 'wasm2c', support_file + '.c')) as f: + total += f.read() # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) @@ -2857,7 +2863,7 @@ def name(i): sig, wabt_sig )) - total = total.replace('/* {{{ EMCC_INVOKE_IMPLS }}} */', '\n'.join(invokes)) + total += '\n'.join(invokes) # write out the final file with open(c_file, 'w') as out: out.write(total) diff --git a/tools/wasm2c/autodebug.c b/tools/wasm2c/autodebug.c new file mode 100644 index 0000000000000..d36c63e9eed10 --- /dev/null +++ b/tools/wasm2c/autodebug.c @@ -0,0 +1,87 @@ +IMPORT_IMPL(void, Z_envZ_log_executionZ_vi, (u32 loc), { + printf("log_execution %d\n", loc); +}); +IMPORT_IMPL(u32, Z_envZ_get_i32Z_iiii, (u32 loc, u32 index, u32 value), { + printf("get_i32 %d,%d,%d\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_get_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { + printf("get_i64 %d,%d,%d,%d\n", loc, index, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_get_f32Z_fiif, (u32 loc, u32 index, f32 value), { + printf("get_f32 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_get_f64Z_diid, (u32 loc, u32 index, f64 value), { + printf("get_f64 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_set_i32Z_iiii, (u32 loc, u32 index, u32 value), { + printf("set_i32 %d,%d,%d\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_set_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { + printf("set_i64 %d,%d,%d,%d\n", loc, index, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(f32, Z_envZ_set_f32Z_fiif, (u32 loc, u32 index, f32 value), { + printf("set_f32 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_set_f64Z_diid, (u32 loc, u32 index, f64 value), { + printf("set_f64 %d,%d,%f\n", loc, index, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_load_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { + printf("load_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); + return ptr; +}); +IMPORT_IMPL(u32, Z_envZ_load_val_i32Z_iii, (u32 loc, u32 value), { + printf("load_val_i32 %d,%d\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_load_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { + printf("load_val_i64 %d,%d,%d\n", loc, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(u64, Z_envZ_load_val_i64Z_jij, (u32 loc, u64 value), { + printf("load_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); + return value; +}); +IMPORT_IMPL(f32, Z_envZ_load_val_f32Z_fif, (u32 loc, f32 value), { + printf("load_val_f32 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_load_val_f64Z_did, (u32 loc, f64 value), { + printf("load_val_f64 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_store_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { + printf("store_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); + return ptr; +}); +IMPORT_IMPL(u32, Z_envZ_store_val_i32Z_iii, (u32 loc, u32 value), { + printf("store_val_i32 %d,%d\n", loc, value); + return value; +}); +IMPORT_IMPL(u32, Z_envZ_store_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { + printf("store_val_i64 %d,%d,%d\n", loc, low, high); + tempRet0 = high; + return low; +}); +IMPORT_IMPL(u64, Z_envZ_store_val_i64Z_jij, (u32 loc, u64 value), { + printf("store_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); + return value; +}); +IMPORT_IMPL(f32, Z_envZ_store_val_f32Z_fif, (u32 loc, f32 value), { + printf("store_val_f32 %d,%f\n", loc, value); + return value; +}); +IMPORT_IMPL(f64, Z_envZ_store_val_f64Z_did, (u32 loc, f64 value), { + printf("store_val_f64 %d,%f\n", loc, value); + return value; +}); diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c new file mode 100644 index 0000000000000..b7e5c94235bde --- /dev/null +++ b/tools/wasm2c/base.c @@ -0,0 +1,183 @@ +/* + * Base of all support for wasm2c code. + */ + +#define __USE_GNU // for O_PATH + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "wasm-rt.h" +#include "wasm-rt-impl.h" + +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#define LIKELY(x) __builtin_expect(!!(x), 1) + +#define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0) + +#define MEMACCESS(addr) ((void*)&Z_memory->data[addr]) + +#undef MEMCHECK +#define MEMCHECK(a, t) \ + if (UNLIKELY((a) + sizeof(t) > Z_memory->size)) TRAP(OOB) + +#undef DEFINE_LOAD +#define DEFINE_LOAD(name, t1, t2, t3) \ + static inline t3 name(u64 addr) { \ + MEMCHECK(addr, t1); \ + t1 result; \ + memcpy(&result, MEMACCESS(addr), sizeof(t1)); \ + return (t3)(t2)result; \ + } + +#undef DEFINE_STORE +#define DEFINE_STORE(name, t1, t2) \ + static inline void name(u64 addr, t2 value) { \ + MEMCHECK(addr, t1); \ + t1 wrapped = (t1)value; \ + memcpy(MEMACCESS(addr), &wrapped, sizeof(t1)); \ + } + +DEFINE_LOAD(wasm_i32_load, u32, u32, u32); +DEFINE_LOAD(wasm_i64_load, u64, u64, u64); +DEFINE_LOAD(wasm_f32_load, f32, f32, f32); +DEFINE_LOAD(wasm_f64_load, f64, f64, f64); +DEFINE_LOAD(wasm_i32_load8_s, s8, s32, u32); +DEFINE_LOAD(wasm_i64_load8_s, s8, s64, u64); +DEFINE_LOAD(wasm_i32_load8_u, u8, u32, u32); +DEFINE_LOAD(wasm_i64_load8_u, u8, u64, u64); +DEFINE_LOAD(wasm_i32_load16_s, s16, s32, u32); +DEFINE_LOAD(wasm_i64_load16_s, s16, s64, u64); +DEFINE_LOAD(wasm_i32_load16_u, u16, u32, u32); +DEFINE_LOAD(wasm_i64_load16_u, u16, u64, u64); +DEFINE_LOAD(wasm_i64_load32_s, s32, s64, u64); +DEFINE_LOAD(wasm_i64_load32_u, u32, u64, u64); +DEFINE_STORE(wasm_i32_store, u32, u32); +DEFINE_STORE(wasm_i64_store, u64, u64); +DEFINE_STORE(wasm_f32_store, f32, f32); +DEFINE_STORE(wasm_f64_store, f64, f64); +DEFINE_STORE(wasm_i32_store8, u8, u32); +DEFINE_STORE(wasm_i32_store16, u16, u32); +DEFINE_STORE(wasm_i64_store8, u8, u64); +DEFINE_STORE(wasm_i64_store16, u16, u64); +DEFINE_STORE(wasm_i64_store32, u32, u64); + +// Imports + +#ifdef VERBOSE_LOGGING +#define VERBOSE_LOG(...) { printf(__VA_ARGS__); } +#else +#define VERBOSE_LOG(...) +#endif + +#define IMPORT_IMPL(ret, name, params, body) \ +ret _##name params { \ + VERBOSE_LOG("[import: " #name "]\n"); \ + body \ +} \ +ret (*name) params = _##name; + +#define STUB_IMPORT_IMPL(ret, name, params, returncode) IMPORT_IMPL(ret, name, params, { return returncode; }); + +// Maintain a stack of setjmps, each jump taking us back to the last invoke. + +#define MAX_SETJMP_STACK 1024 + +static jmp_buf setjmp_stack[MAX_SETJMP_STACK]; + +static u32 next_setjmp = 0; + +// Declare exports for invokes. We should generate them based on what the +// wasm needs, but for now have a fixed list here. To get things to link, +// declare them, so they either link with the existing value in the main +// wasm2c .c output file, or else they contain NULL but will never be called. + +#define DECLARE_EXPORT(ret, name, args) \ +__attribute__((weak)) \ +ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; + +DECLARE_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); + +// Stack support should be linked in if it is needed. +IMPORT_IMPL(__attribute__((weak)) u32, Z_stackSaveZ_iv, (), { + abort(); +}); +IMPORT_IMPL(__attribute__((weak))void, Z_stackRestoreZ_vi, (u32 x), { + abort(); +}); + +#define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ +DECLARE_EXPORT(void, dyncall, types); \ +\ +IMPORT_IMPL(void, name, typed_args, { \ + VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ + u32 sp = Z_stackSaveZ_iv(); \ + if (next_setjmp >= MAX_SETJMP_STACK) { \ + abort_with_message("too many nested setjmps"); \ + } \ + u32 id = next_setjmp++; \ + int result = setjmp(setjmp_stack[id]); \ + if (result == 0) { \ + (* dyncall) args; \ + /* if we got here, no longjmp or exception happened, we returned normally */ \ + } else { \ + /* A longjmp or an exception took us here. */ \ + Z_stackRestoreZ_vi(sp); \ + Z_setThrewZ_vii(1, 0); \ + } \ + next_setjmp--; \ +}); + +#define RETURNING_INVOKE_IMPL(ret, name, typed_args, types, args, dyncall) \ +DECLARE_EXPORT(ret, dyncall, types); \ +\ +IMPORT_IMPL(ret, name, typed_args, { \ + VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ + u32 sp = Z_stackSaveZ_iv(); \ + if (next_setjmp >= MAX_SETJMP_STACK) { \ + abort_with_message("too many nested setjmps"); \ + } \ + u32 id = next_setjmp++; \ + int result = setjmp(setjmp_stack[id]); \ + ret returned_value = 0; \ + if (result == 0) { \ + returned_value = (* dyncall) args; \ + /* if we got here, no longjmp or exception happened, we returned normally */ \ + } else { \ + /* A longjmp or an exception took us here. */ \ + Z_stackRestoreZ_vi(sp); \ + Z_setThrewZ_vii(1, 0); \ + } \ + next_setjmp--; \ + return returned_value; \ +}); + +IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { + if (next_setjmp == 0) { + abort_with_message("longjmp without setjmp"); + } + Z_setThrewZ_vii(buf, value ? value : 1); + longjmp(setjmp_stack[next_setjmp - 1], 1); +}); + +IMPORT_IMPL(void, Z_envZ_emscripten_notify_memory_growthZ_vi, (u32 size), {}); + +static u32 tempRet0 = 0; + +IMPORT_IMPL(u32, Z_envZ_getTempRet0Z_iv, (), { + return tempRet0; +}); + +IMPORT_IMPL(void, Z_envZ_setTempRet0Z_vi, (u32 x), { + tempRet0 = x; +}); diff --git a/tools/wasm2c/main.c b/tools/wasm2c/main.c index 3ff1983ac3094..1ff1108468e26 100644 --- a/tools/wasm2c/main.c +++ b/tools/wasm2c/main.c @@ -1,377 +1,3 @@ -/* - * A main file to run wasm2c code. This implements various wasi and emscripten - * syscalls, and allows direct/unsandboxed file access TODO add options - */ - -#define __USE_GNU // for O_PATH - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "wasm-rt.h" -#include "wasm-rt-impl.h" - -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#define LIKELY(x) __builtin_expect(!!(x), 1) - -#define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0) - -#define MEMACCESS(addr) ((void*)&Z_memory->data[addr]) - -#undef MEMCHECK -#define MEMCHECK(a, t) \ - if (UNLIKELY((a) + sizeof(t) > Z_memory->size)) TRAP(OOB) - -#undef DEFINE_LOAD -#define DEFINE_LOAD(name, t1, t2, t3) \ - static inline t3 name(u64 addr) { \ - MEMCHECK(addr, t1); \ - t1 result; \ - memcpy(&result, MEMACCESS(addr), sizeof(t1)); \ - return (t3)(t2)result; \ - } - -#undef DEFINE_STORE -#define DEFINE_STORE(name, t1, t2) \ - static inline void name(u64 addr, t2 value) { \ - MEMCHECK(addr, t1); \ - t1 wrapped = (t1)value; \ - memcpy(MEMACCESS(addr), &wrapped, sizeof(t1)); \ - } - -DEFINE_LOAD(wasm_i32_load, u32, u32, u32); -DEFINE_LOAD(wasm_i64_load, u64, u64, u64); -DEFINE_LOAD(wasm_f32_load, f32, f32, f32); -DEFINE_LOAD(wasm_f64_load, f64, f64, f64); -DEFINE_LOAD(wasm_i32_load8_s, s8, s32, u32); -DEFINE_LOAD(wasm_i64_load8_s, s8, s64, u64); -DEFINE_LOAD(wasm_i32_load8_u, u8, u32, u32); -DEFINE_LOAD(wasm_i64_load8_u, u8, u64, u64); -DEFINE_LOAD(wasm_i32_load16_s, s16, s32, u32); -DEFINE_LOAD(wasm_i64_load16_s, s16, s64, u64); -DEFINE_LOAD(wasm_i32_load16_u, u16, u32, u32); -DEFINE_LOAD(wasm_i64_load16_u, u16, u64, u64); -DEFINE_LOAD(wasm_i64_load32_s, s32, s64, u64); -DEFINE_LOAD(wasm_i64_load32_u, u32, u64, u64); -DEFINE_STORE(wasm_i32_store, u32, u32); -DEFINE_STORE(wasm_i64_store, u64, u64); -DEFINE_STORE(wasm_f32_store, f32, f32); -DEFINE_STORE(wasm_f64_store, f64, f64); -DEFINE_STORE(wasm_i32_store8, u8, u32); -DEFINE_STORE(wasm_i32_store16, u16, u32); -DEFINE_STORE(wasm_i64_store8, u8, u64); -DEFINE_STORE(wasm_i64_store16, u16, u64); -DEFINE_STORE(wasm_i64_store32, u32, u64); - -// Imports - -#ifdef VERBOSE_LOGGING -#define VERBOSE_LOG(...) { printf(__VA_ARGS__); } -#else -#define VERBOSE_LOG(...) -#endif - -#define IMPORT_IMPL(ret, name, params, body) \ -ret _##name params { \ - VERBOSE_LOG("[import: " #name "]\n"); \ - body \ -} \ -ret (*name) params = _##name; - -#define STUB_IMPORT_IMPL(ret, name, params, returncode) IMPORT_IMPL(ret, name, params, { return returncode; }); - -#define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ -#define WASI_EINVAL 28 - -IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { - exit(x); -}); - -#define MAX_FDS 1024 - -static int wasm_fd_to_native[MAX_FDS]; - -static u32 next_wasm_fd; - -static void init_fds() { - wasm_fd_to_native[0] = STDIN_FILENO; - wasm_fd_to_native[1] = STDOUT_FILENO; - wasm_fd_to_native[2] = STDERR_FILENO; - next_wasm_fd = 3; -} - -void abort_with_message(const char* message) { - fprintf(stderr, "%s\n", message); - abort(); -} - -static u32 get_or_allocate_wasm_fd(int nfd) { - // If the native fd is already mapped, return the same wasm fd for it. - for (int i = 0; i < next_wasm_fd; i++) { - if (wasm_fd_to_native[i] == nfd) { - return i; - } - } - if (next_wasm_fd >= MAX_FDS) { - abort_with_message("ran out of fds"); - } - u32 fd = next_wasm_fd; - wasm_fd_to_native[fd] = nfd; - next_wasm_fd++; - return fd; -} - -static int get_native_fd(u32 fd) { - if (fd >= MAX_FDS || fd >= next_wasm_fd) { - return -1; - } - return wasm_fd_to_native[fd]; -} - -IMPORT_IMPL(u32, Z_envZ___sys_openZ_iiii, (u32 path, u32 flags, u32 varargs), { - VERBOSE_LOG(" open: %s %d %d\n", MEMACCESS(path), flags, wasm_i32_load(varargs)); - int nfd = open(MEMACCESS(path), flags, wasm_i32_load(varargs)); - VERBOSE_LOG(" => native %d\n", nfd); - if (nfd >= 0) { - u32 fd = get_or_allocate_wasm_fd(nfd); - VERBOSE_LOG(" => wasm %d\n", fd); - return fd; - } - return -1; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { - int nfd = get_native_fd(fd); - VERBOSE_LOG(" fd_write wasm %d => native %d\n", fd, nfd); - if (nfd < 0) { - return WASI_DEFAULT_ERROR; - } - u32 num = 0; - for (u32 i = 0; i < iovcnt; i++) { - u32 ptr = wasm_i32_load(iov + i * 8); - u32 len = wasm_i32_load(iov + i * 8 + 4); - VERBOSE_LOG(" chunk %d %d\n", ptr, len); - ssize_t result; - // Use stdio for stdout/stderr to avoid mixing a low-level write() with - // other logging code, which can change the order from the expected. - if (nfd == STDOUT_FILENO) { - result = fwrite(MEMACCESS(ptr), 1, len, stdout); - } else if (nfd == STDERR_FILENO) { - result = fwrite(MEMACCESS(ptr), 1, len, stderr); - } else { - result = write(nfd, MEMACCESS(ptr), len); - } - if (result < 0) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return WASI_DEFAULT_ERROR; - } - if (result != len) { - VERBOSE_LOG(" amount error, %ld %d\n", result, len); - return WASI_DEFAULT_ERROR; - } - num += len; - } - VERBOSE_LOG(" success: %d\n", num); - wasm_i32_store(pnum, num); - return 0; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { - int nfd = get_native_fd(fd); - VERBOSE_LOG(" fd_read wasm %d => native %d\n", fd, nfd); - if (nfd < 0) { - return WASI_DEFAULT_ERROR; - } - u32 num = 0; - for (u32 i = 0; i < iovcnt; i++) { - u32 ptr = wasm_i32_load(iov + i * 8); - u32 len = wasm_i32_load(iov + i * 8 + 4); - VERBOSE_LOG(" chunk %d %d\n", ptr, len); - ssize_t result = read(nfd, MEMACCESS(ptr), len); - if (result < 0) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return WASI_DEFAULT_ERROR; - } - num += result; - if (result != len) { - break; // nothing more to read - } - } - VERBOSE_LOG(" success: %d\n", num); - wasm_i32_store(pnum, num); - return 0; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_closeZ_ii, (u32 fd), { - // TODO full file support - int nfd = get_native_fd(fd); - VERBOSE_LOG(" close wasm %d => native %d\n", fd, nfd); - if (nfd < 0) { - return WASI_DEFAULT_ERROR; - } - close(nfd); - return 0; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_sizes_getZ_iii, (u32 pcount, u32 pbuf_size), { - // TODO: connect to actual env? - wasm_i32_store(pcount, 0); - wasm_i32_store(pbuf_size, 0); - return 0; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_getZ_iii, (u32 __environ, u32 environ_buf), { - // TODO: connect to actual env? - return 0; -}); - -static int whence_to_native(u32 whence) { - if (whence == 0) return SEEK_SET; - if (whence == 1) return SEEK_CUR; - if (whence == 2) return SEEK_END; - return -1; -} - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, u32 whence, u32 new_offset), { - int nfd = get_native_fd(fd); - int nwhence = whence_to_native(whence); - VERBOSE_LOG(" seek %d (=> native %d) %ld %d (=> %d) %d\n", fd, nfd, offset, whence, nwhence, new_offset); - if (nfd < 0) { - return WASI_DEFAULT_ERROR; - } - off_t off = lseek(nfd, offset, nwhence); - VERBOSE_LOG(" off: %ld\n", off); - if (off == (off_t)-1) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return WASI_DEFAULT_ERROR; - } - wasm_i64_store(new_offset, off); - return 0; -}); -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), { - return Z_wasi_snapshot_preview1Z_fd_seekZ_iijii(a, b + (((u64)c) << 32), d, e); -}); -STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_fdstat_getZ_iii, (u32 a, u32 b), WASI_DEFAULT_ERROR); -STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_syncZ_ii, (u32 a), WASI_DEFAULT_ERROR); - -// TODO: set errno in wasm for everything - -STUB_IMPORT_IMPL(u32, Z_envZ_dlopenZ_iii, (u32 a, u32 b), 1); -STUB_IMPORT_IMPL(u32, Z_envZ_dlcloseZ_ii, (u32 a), 1); -STUB_IMPORT_IMPL(u32, Z_envZ_dlsymZ_iii, (u32 a, u32 b), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_dlerrorZ_iv, (), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_signalZ_iii, (u32 a, u32 b), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_systemZ_ii, (u32 a), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_utimesZ_iii, (u32 a, u32 b), -1); - -// Syscalls return a negative error code -#define EM_EACCES -2 - -IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), { - VERBOSE_LOG(" unlink %s\n", MEMACCESS(path)); - if (unlink(MEMACCESS(path))) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return EM_EACCES; - } - return 0; -}); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_rmdirZ_ii, (u32 a), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_renameZ_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_lstat64Z_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup3Z_iiii, (u32 a, u32 b, u32 c), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup2Z_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_getcwdZ_iii, (u32 a, u32 b), EM_EACCES); - -static u32 do_stat(int nfd, u32 buf) { - struct stat nbuf; - if (fstat(nfd, &nbuf)) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return EM_EACCES; - } - VERBOSE_LOG(" success, size=%ld\n", nbuf.st_size); - wasm_i32_store(buf + 0, nbuf.st_dev); - wasm_i32_store(buf + 4, 0); - wasm_i32_store(buf + 8, nbuf.st_ino); - wasm_i32_store(buf + 12, nbuf.st_mode); - wasm_i32_store(buf + 16, nbuf.st_nlink); - wasm_i32_store(buf + 20, nbuf.st_uid); - wasm_i32_store(buf + 24, nbuf.st_gid); - wasm_i32_store(buf + 28, nbuf.st_rdev); - wasm_i32_store(buf + 32, 0); - wasm_i64_store(buf + 40, nbuf.st_size); - wasm_i32_store(buf + 48, nbuf.st_blksize); - wasm_i32_store(buf + 52, nbuf.st_blocks); - wasm_i32_store(buf + 56, nbuf.st_atim.tv_sec); - wasm_i32_store(buf + 60, nbuf.st_atim.tv_nsec); - wasm_i32_store(buf + 64, nbuf.st_mtim.tv_sec); - wasm_i32_store(buf + 68, nbuf.st_mtim.tv_nsec); - wasm_i32_store(buf + 72, nbuf.st_ctim.tv_sec); - wasm_i32_store(buf + 76, nbuf.st_ctim.tv_nsec); - wasm_i64_store(buf + 80, nbuf.st_ino); - return 0; -} - -IMPORT_IMPL(u32, Z_envZ___sys_fstat64Z_iii, (u32 fd, u32 buf), { - int nfd = get_native_fd(fd); - VERBOSE_LOG(" fstat64 %d (=> %d) %d\n", fd, nfd, buf); - if (nfd < 0) { - return EM_EACCES; - } - return do_stat(nfd, buf); -}); - -IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), { - VERBOSE_LOG(" stat64: %s\n", MEMACCESS(path)); - int nfd = open(MEMACCESS(path), O_PATH); - if (nfd < 0) { - VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); - return EM_EACCES; - } - return do_stat(nfd, buf); -}); - -STUB_IMPORT_IMPL(u32, Z_envZ___sys_ftruncate64Z_iiiii, (u32 a, u32 b, u32 c, u32 d), EM_EACCES); -IMPORT_IMPL(u32, Z_envZ___sys_readZ_iiii, (u32 fd, u32 buf, u32 count), { - int nfd = get_native_fd(fd); - VERBOSE_LOG(" read %d (=> %d) %d %d\n", fd, nfd, buf, count); - if (nfd < 0) { - VERBOSE_LOG(" bad fd\n"); - return EM_EACCES; - } - ssize_t ret = read(nfd, MEMACCESS(buf), count); - VERBOSE_LOG(" native read: %ld\n", ret); - if (ret < 0) { - VERBOSE_LOG(" read error %d %s\n", errno, strerror(errno)); - return EM_EACCES; - } - return ret; -}); - -IMPORT_IMPL(u32, Z_envZ___sys_accessZ_iii, (u32 pathname, u32 mode), { - VERBOSE_LOG(" access: %s 0x%x\n", MEMACCESS(pathname), mode); - // TODO: sandboxing, convert mode - int result = access(MEMACCESS(pathname), mode); - if (result < 0) { - VERBOSE_LOG(" access error: %d %s\n", errno, strerror(errno)); - return EM_EACCES; - } - return 0; -}); - -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_initZ_ii, (u32 a), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_settypeZ_iii, (u32 a, u32 b), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_destroyZ_ii, (u32 a), 0); - static int main_argc; static char** main_argv; @@ -397,229 +23,6 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_args_getZ_iii, (u32 argv, u32 argv_bu return 0; }); -// Maintain a stack of setjmps, each jump taking us back to the last invoke. - -#define MAX_SETJMP_STACK 1024 - -static jmp_buf setjmp_stack[MAX_SETJMP_STACK]; - -static u32 next_setjmp = 0; - -// Declare exports for invokes. We should generate them based on what the -// wasm needs, but for now have a fixed list here. To get things to link, -// declare them, so they either link with the existing value in the main -// wasm2c .c output file, or else they contain NULL but will never be called. - -#define DECLARE_EXPORT(ret, name, args) \ -__attribute__((weak)) \ -ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; - -DECLARE_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); - -// Stack support should be linked in if it is needed. -IMPORT_IMPL(__attribute__((weak)) u32, Z_stackSaveZ_iv, (), { - abort(); -}); -IMPORT_IMPL(__attribute__((weak))void, Z_stackRestoreZ_vi, (u32 x), { - abort(); -}); - -#define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ -DECLARE_EXPORT(void, dyncall, types); \ -\ -IMPORT_IMPL(void, name, typed_args, { \ - VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ - u32 sp = Z_stackSaveZ_iv(); \ - if (next_setjmp >= MAX_SETJMP_STACK) { \ - abort_with_message("too many nested setjmps"); \ - } \ - u32 id = next_setjmp++; \ - int result = setjmp(setjmp_stack[id]); \ - if (result == 0) { \ - (* dyncall) args; \ - /* if we got here, no longjmp or exception happened, we returned normally */ \ - } else { \ - /* A longjmp or an exception took us here. */ \ - Z_stackRestoreZ_vi(sp); \ - Z_setThrewZ_vii(1, 0); \ - } \ - next_setjmp--; \ -}); - -#define RETURNING_INVOKE_IMPL(ret, name, typed_args, types, args, dyncall) \ -DECLARE_EXPORT(ret, dyncall, types); \ -\ -IMPORT_IMPL(ret, name, typed_args, { \ - VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ - u32 sp = Z_stackSaveZ_iv(); \ - if (next_setjmp >= MAX_SETJMP_STACK) { \ - abort_with_message("too many nested setjmps"); \ - } \ - u32 id = next_setjmp++; \ - int result = setjmp(setjmp_stack[id]); \ - ret returned_value = 0; \ - if (result == 0) { \ - returned_value = (* dyncall) args; \ - /* if we got here, no longjmp or exception happened, we returned normally */ \ - } else { \ - /* A longjmp or an exception took us here. */ \ - Z_stackRestoreZ_vi(sp); \ - Z_setThrewZ_vii(1, 0); \ - } \ - next_setjmp--; \ - return returned_value; \ -}); - -/* {{{ EMCC_INVOKE_IMPLS }}} */ - -IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { - if (next_setjmp == 0) { - abort_with_message("longjmp without setjmp"); - } - Z_setThrewZ_vii(buf, value ? value : 1); - longjmp(setjmp_stack[next_setjmp - 1], 1); -}); - -#define WASM_CLOCK_REALTIME 0 -#define WASM_CLOCK_MONOTONIC 1 -#define WASM_CLOCK_PROCESS_CPUTIME 2 -#define WASM_CLOCK_CLOCK_THREAD_CPUTIME_ID 3 - -static int check_clock(u32 clock_id) { - return clock_id == WASM_CLOCK_REALTIME || clock_id == WASM_CLOCK_MONOTONIC || - clock_id == WASM_CLOCK_PROCESS_CPUTIME || clock_id == CLOCK_THREAD_CPUTIME_ID; -} - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { - if (!check_clock(clock_id)) { - return WASI_EINVAL; - } - // TODO: handle realtime vs monotonic etc. - // wasi expects a result in nanoseconds, and we know how to convert clock() - // to seconds, so compute from there - const double NSEC_PER_SEC = 1000.0 * 1000.0 * 1000.0; - wasm_i64_store(out, (u64)(clock() / (CLOCKS_PER_SEC / NSEC_PER_SEC))); - return 0; -}); - -IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), { - if (!check_clock(clock_id)) { - return WASI_EINVAL; - } - // TODO: handle realtime vs monotonic etc. For now just report "milliseconds". - wasm_i64_store(out, 1000 * 1000); - return 0; -}); - -IMPORT_IMPL(void, Z_envZ_emscripten_notify_memory_growthZ_vi, (u32 size), {}); - -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_joinZ_iii, (u32 a, u32 b), -1); - -STUB_IMPORT_IMPL(u32, Z_envZ___cxa_thread_atexitZ_iiii, (u32 a, u32 b, u32 c), -1); - -static u32 tempRet0 = 0; - -IMPORT_IMPL(u32, Z_envZ_getTempRet0Z_iv, (), { - return tempRet0; -}); - -IMPORT_IMPL(void, Z_envZ_setTempRet0Z_vi, (u32 x), { - tempRet0 = x; -}); - -// autodebug - -IMPORT_IMPL(void, Z_envZ_log_executionZ_vi, (u32 loc), { - printf("log_execution %d\n", loc); -}); -IMPORT_IMPL(u32, Z_envZ_get_i32Z_iiii, (u32 loc, u32 index, u32 value), { - printf("get_i32 %d,%d,%d\n", loc, index, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_get_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { - printf("get_i64 %d,%d,%d,%d\n", loc, index, low, high); - tempRet0 = high; - return low; -}); -IMPORT_IMPL(f32, Z_envZ_get_f32Z_fiif, (u32 loc, u32 index, f32 value), { - printf("get_f32 %d,%d,%f\n", loc, index, value); - return value; -}); -IMPORT_IMPL(f64, Z_envZ_get_f64Z_diid, (u32 loc, u32 index, f64 value), { - printf("get_f64 %d,%d,%f\n", loc, index, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_set_i32Z_iiii, (u32 loc, u32 index, u32 value), { - printf("set_i32 %d,%d,%d\n", loc, index, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_set_i64Z_iiiii, (u32 loc, u32 index, u32 low, u32 high), { - printf("set_i64 %d,%d,%d,%d\n", loc, index, low, high); - tempRet0 = high; - return low; -}); -IMPORT_IMPL(f32, Z_envZ_set_f32Z_fiif, (u32 loc, u32 index, f32 value), { - printf("set_f32 %d,%d,%f\n", loc, index, value); - return value; -}); -IMPORT_IMPL(f64, Z_envZ_set_f64Z_diid, (u32 loc, u32 index, f64 value), { - printf("set_f64 %d,%d,%f\n", loc, index, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_load_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { - printf("load_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); - return ptr; -}); -IMPORT_IMPL(u32, Z_envZ_load_val_i32Z_iii, (u32 loc, u32 value), { - printf("load_val_i32 %d,%d\n", loc, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_load_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { - printf("load_val_i64 %d,%d,%d\n", loc, low, high); - tempRet0 = high; - return low; -}); -IMPORT_IMPL(u64, Z_envZ_load_val_i64Z_jij, (u32 loc, u64 value), { - printf("load_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); - return value; -}); -IMPORT_IMPL(f32, Z_envZ_load_val_f32Z_fif, (u32 loc, f32 value), { - printf("load_val_f32 %d,%f\n", loc, value); - return value; -}); -IMPORT_IMPL(f64, Z_envZ_load_val_f64Z_did, (u32 loc, f64 value), { - printf("load_val_f64 %d,%f\n", loc, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_store_ptrZ_iiiii, (u32 loc, u32 bytes, u32 offset, u32 ptr), { - printf("store_ptr %d,%d,%d,%d\n", loc, bytes, offset, ptr); - return ptr; -}); -IMPORT_IMPL(u32, Z_envZ_store_val_i32Z_iii, (u32 loc, u32 value), { - printf("store_val_i32 %d,%d\n", loc, value); - return value; -}); -IMPORT_IMPL(u32, Z_envZ_store_val_i64Z_iiii, (u32 loc, u32 low, u32 high), { - printf("store_val_i64 %d,%d,%d\n", loc, low, high); - tempRet0 = high; - return low; -}); -IMPORT_IMPL(u64, Z_envZ_store_val_i64Z_jij, (u32 loc, u64 value), { - printf("store_val_i64 %d,%d,%d\n", loc, (u32)value, (u32)(value >> 32)); - return value; -}); -IMPORT_IMPL(f32, Z_envZ_store_val_f32Z_fif, (u32 loc, f32 value), { - printf("store_val_f32 %d,%f\n", loc, value); - return value; -}); -IMPORT_IMPL(f64, Z_envZ_store_val_f64Z_did, (u32 loc, f64 value), { - printf("store_val_f64 %d,%f\n", loc, value); - return value; -}); - -// Main - int main(int argc, char** argv) { main_argc = argc; main_argv = argv; diff --git a/tools/wasm2c/os.c b/tools/wasm2c/os.c new file mode 100644 index 0000000000000..2f2d58eee520f --- /dev/null +++ b/tools/wasm2c/os.c @@ -0,0 +1,316 @@ +#define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ +#define WASI_EINVAL 28 + +IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { + exit(x); +}); + +#define MAX_FDS 1024 + +static int wasm_fd_to_native[MAX_FDS]; + +static u32 next_wasm_fd; + +static void init_fds() { + wasm_fd_to_native[0] = STDIN_FILENO; + wasm_fd_to_native[1] = STDOUT_FILENO; + wasm_fd_to_native[2] = STDERR_FILENO; + next_wasm_fd = 3; +} + +void abort_with_message(const char* message) { + fprintf(stderr, "%s\n", message); + abort(); +} + +static u32 get_or_allocate_wasm_fd(int nfd) { + // If the native fd is already mapped, return the same wasm fd for it. + for (int i = 0; i < next_wasm_fd; i++) { + if (wasm_fd_to_native[i] == nfd) { + return i; + } + } + if (next_wasm_fd >= MAX_FDS) { + abort_with_message("ran out of fds"); + } + u32 fd = next_wasm_fd; + wasm_fd_to_native[fd] = nfd; + next_wasm_fd++; + return fd; +} + +static int get_native_fd(u32 fd) { + if (fd >= MAX_FDS || fd >= next_wasm_fd) { + return -1; + } + return wasm_fd_to_native[fd]; +} + +IMPORT_IMPL(u32, Z_envZ___sys_openZ_iiii, (u32 path, u32 flags, u32 varargs), { + VERBOSE_LOG(" open: %s %d %d\n", MEMACCESS(path), flags, wasm_i32_load(varargs)); + int nfd = open(MEMACCESS(path), flags, wasm_i32_load(varargs)); + VERBOSE_LOG(" => native %d\n", nfd); + if (nfd >= 0) { + u32 fd = get_or_allocate_wasm_fd(nfd); + VERBOSE_LOG(" => wasm %d\n", fd); + return fd; + } + return -1; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fd_write wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + u32 num = 0; + for (u32 i = 0; i < iovcnt; i++) { + u32 ptr = wasm_i32_load(iov + i * 8); + u32 len = wasm_i32_load(iov + i * 8 + 4); + VERBOSE_LOG(" chunk %d %d\n", ptr, len); + ssize_t result; + // Use stdio for stdout/stderr to avoid mixing a low-level write() with + // other logging code, which can change the order from the expected. + if (nfd == STDOUT_FILENO) { + result = fwrite(MEMACCESS(ptr), 1, len, stdout); + } else if (nfd == STDERR_FILENO) { + result = fwrite(MEMACCESS(ptr), 1, len, stderr); + } else { + result = write(nfd, MEMACCESS(ptr), len); + } + if (result < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + if (result != len) { + VERBOSE_LOG(" amount error, %ld %d\n", result, len); + return WASI_DEFAULT_ERROR; + } + num += len; + } + VERBOSE_LOG(" success: %d\n", num); + wasm_i32_store(pnum, num); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fd_read wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + u32 num = 0; + for (u32 i = 0; i < iovcnt; i++) { + u32 ptr = wasm_i32_load(iov + i * 8); + u32 len = wasm_i32_load(iov + i * 8 + 4); + VERBOSE_LOG(" chunk %d %d\n", ptr, len); + ssize_t result = read(nfd, MEMACCESS(ptr), len); + if (result < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + num += result; + if (result != len) { + break; // nothing more to read + } + } + VERBOSE_LOG(" success: %d\n", num); + wasm_i32_store(pnum, num); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_closeZ_ii, (u32 fd), { + // TODO full file support + int nfd = get_native_fd(fd); + VERBOSE_LOG(" close wasm %d => native %d\n", fd, nfd); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + close(nfd); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_sizes_getZ_iii, (u32 pcount, u32 pbuf_size), { + // TODO: connect to actual env? + wasm_i32_store(pcount, 0); + wasm_i32_store(pbuf_size, 0); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_getZ_iii, (u32 __environ, u32 environ_buf), { + // TODO: connect to actual env? + return 0; +}); + +static int whence_to_native(u32 whence) { + if (whence == 0) return SEEK_SET; + if (whence == 1) return SEEK_CUR; + if (whence == 2) return SEEK_END; + return -1; +} + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, u32 whence, u32 new_offset), { + int nfd = get_native_fd(fd); + int nwhence = whence_to_native(whence); + VERBOSE_LOG(" seek %d (=> native %d) %ld %d (=> %d) %d\n", fd, nfd, offset, whence, nwhence, new_offset); + if (nfd < 0) { + return WASI_DEFAULT_ERROR; + } + off_t off = lseek(nfd, offset, nwhence); + VERBOSE_LOG(" off: %ld\n", off); + if (off == (off_t)-1) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return WASI_DEFAULT_ERROR; + } + wasm_i64_store(new_offset, off); + return 0; +}); +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), { + return Z_wasi_snapshot_preview1Z_fd_seekZ_iijii(a, b + (((u64)c) << 32), d, e); +}); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_fdstat_getZ_iii, (u32 a, u32 b), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_syncZ_ii, (u32 a), WASI_DEFAULT_ERROR); + +// TODO: set errno in wasm for everything + +STUB_IMPORT_IMPL(u32, Z_envZ_dlopenZ_iii, (u32 a, u32 b), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlcloseZ_ii, (u32 a), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlsymZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_dlerrorZ_iv, (), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_signalZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_systemZ_ii, (u32 a), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_utimesZ_iii, (u32 a, u32 b), -1); + +// Syscalls return a negative error code +#define EM_EACCES -2 + +IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), { + VERBOSE_LOG(" unlink %s\n", MEMACCESS(path)); + if (unlink(MEMACCESS(path))) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return 0; +}); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_rmdirZ_ii, (u32 a), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_renameZ_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_lstat64Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup3Z_iiii, (u32 a, u32 b, u32 c), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup2Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_getcwdZ_iii, (u32 a, u32 b), EM_EACCES); + +static u32 do_stat(int nfd, u32 buf) { + struct stat nbuf; + if (fstat(nfd, &nbuf)) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + VERBOSE_LOG(" success, size=%ld\n", nbuf.st_size); + wasm_i32_store(buf + 0, nbuf.st_dev); + wasm_i32_store(buf + 4, 0); + wasm_i32_store(buf + 8, nbuf.st_ino); + wasm_i32_store(buf + 12, nbuf.st_mode); + wasm_i32_store(buf + 16, nbuf.st_nlink); + wasm_i32_store(buf + 20, nbuf.st_uid); + wasm_i32_store(buf + 24, nbuf.st_gid); + wasm_i32_store(buf + 28, nbuf.st_rdev); + wasm_i32_store(buf + 32, 0); + wasm_i64_store(buf + 40, nbuf.st_size); + wasm_i32_store(buf + 48, nbuf.st_blksize); + wasm_i32_store(buf + 52, nbuf.st_blocks); + wasm_i32_store(buf + 56, nbuf.st_atim.tv_sec); + wasm_i32_store(buf + 60, nbuf.st_atim.tv_nsec); + wasm_i32_store(buf + 64, nbuf.st_mtim.tv_sec); + wasm_i32_store(buf + 68, nbuf.st_mtim.tv_nsec); + wasm_i32_store(buf + 72, nbuf.st_ctim.tv_sec); + wasm_i32_store(buf + 76, nbuf.st_ctim.tv_nsec); + wasm_i64_store(buf + 80, nbuf.st_ino); + return 0; +} + +IMPORT_IMPL(u32, Z_envZ___sys_fstat64Z_iii, (u32 fd, u32 buf), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" fstat64 %d (=> %d) %d\n", fd, nfd, buf); + if (nfd < 0) { + return EM_EACCES; + } + return do_stat(nfd, buf); +}); + +IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), { + VERBOSE_LOG(" stat64: %s\n", MEMACCESS(path)); + int nfd = open(MEMACCESS(path), O_PATH); + if (nfd < 0) { + VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return do_stat(nfd, buf); +}); + +STUB_IMPORT_IMPL(u32, Z_envZ___sys_ftruncate64Z_iiiii, (u32 a, u32 b, u32 c, u32 d), EM_EACCES); +IMPORT_IMPL(u32, Z_envZ___sys_readZ_iiii, (u32 fd, u32 buf, u32 count), { + int nfd = get_native_fd(fd); + VERBOSE_LOG(" read %d (=> %d) %d %d\n", fd, nfd, buf, count); + if (nfd < 0) { + VERBOSE_LOG(" bad fd\n"); + return EM_EACCES; + } + ssize_t ret = read(nfd, MEMACCESS(buf), count); + VERBOSE_LOG(" native read: %ld\n", ret); + if (ret < 0) { + VERBOSE_LOG(" read error %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return ret; +}); + +IMPORT_IMPL(u32, Z_envZ___sys_accessZ_iii, (u32 pathname, u32 mode), { + VERBOSE_LOG(" access: %s 0x%x\n", MEMACCESS(pathname), mode); + // TODO: sandboxing, convert mode + int result = access(MEMACCESS(pathname), mode); + if (result < 0) { + VERBOSE_LOG(" access error: %d %s\n", errno, strerror(errno)); + return EM_EACCES; + } + return 0; +}); + +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_initZ_ii, (u32 a), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_settypeZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_destroyZ_ii, (u32 a), 0); + +#define WASM_CLOCK_REALTIME 0 +#define WASM_CLOCK_MONOTONIC 1 +#define WASM_CLOCK_PROCESS_CPUTIME 2 +#define WASM_CLOCK_CLOCK_THREAD_CPUTIME_ID 3 + +static int check_clock(u32 clock_id) { + return clock_id == WASM_CLOCK_REALTIME || clock_id == WASM_CLOCK_MONOTONIC || + clock_id == WASM_CLOCK_PROCESS_CPUTIME || clock_id == CLOCK_THREAD_CPUTIME_ID; +} + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), { + if (!check_clock(clock_id)) { + return WASI_EINVAL; + } + // TODO: handle realtime vs monotonic etc. + // wasi expects a result in nanoseconds, and we know how to convert clock() + // to seconds, so compute from there + const double NSEC_PER_SEC = 1000.0 * 1000.0 * 1000.0; + wasm_i64_store(out, (u64)(clock() / (CLOCKS_PER_SEC / NSEC_PER_SEC))); + return 0; +}); + +IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), { + if (!check_clock(clock_id)) { + return WASI_EINVAL; + } + // TODO: handle realtime vs monotonic etc. For now just report "milliseconds". + wasm_i64_store(out, 1000 * 1000); + return 0; +}); + +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_joinZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ___cxa_thread_atexitZ_iiii, (u32 a, u32 b, u32 c), -1); From 9587ea2563fba69e893d62f7239b479f884810b1 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 13:49:39 -0700 Subject: [PATCH 31/49] works again --- tools/shared.py | 3 +++ tools/wasm2c/base.c | 7 +++++++ tools/wasm2c/os.c | 5 ----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index fb872a478b621..e214f4e31fcb8 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2820,6 +2820,9 @@ def do_wasm2c(infile): if Settings.AUTODEBUG: support_files.append('autodebug') if Settings.EXPECT_MAIN: + # TODO: add an option for direct OS access. For now, do that when building + # an executable with main, as opposed to a library + support_files.append('os') support_files.append('main') for support_file in support_files: with open(path_from_root('tools', 'wasm2c', support_file + '.c')) as f: diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c index b7e5c94235bde..a777007505e67 100644 --- a/tools/wasm2c/base.c +++ b/tools/wasm2c/base.c @@ -89,6 +89,13 @@ ret (*name) params = _##name; #define STUB_IMPORT_IMPL(ret, name, params, returncode) IMPORT_IMPL(ret, name, params, { return returncode; }); +// Generic abort method for a runtime error in the runtime. + +static void abort_with_message(const char* message) { + fprintf(stderr, "%s\n", message); + abort(); +} + // Maintain a stack of setjmps, each jump taking us back to the last invoke. #define MAX_SETJMP_STACK 1024 diff --git a/tools/wasm2c/os.c b/tools/wasm2c/os.c index 2f2d58eee520f..d029f8e393173 100644 --- a/tools/wasm2c/os.c +++ b/tools/wasm2c/os.c @@ -18,11 +18,6 @@ static void init_fds() { next_wasm_fd = 3; } -void abort_with_message(const char* message) { - fprintf(stderr, "%s\n", message); - abort(); -} - static u32 get_or_allocate_wasm_fd(int nfd) { // If the native fd is already mapped, return the same wasm fd for it. for (int i = 0; i < next_wasm_fd; i++) { From 8592f8525a14764c6ecab8cf9194e421c472d929 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 13:52:29 -0700 Subject: [PATCH 32/49] avoid O_PATH [ci skip] --- tools/wasm2c/base.c | 2 -- tools/wasm2c/os.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c index a777007505e67..a2bc8f717ddfa 100644 --- a/tools/wasm2c/base.c +++ b/tools/wasm2c/base.c @@ -2,8 +2,6 @@ * Base of all support for wasm2c code. */ -#define __USE_GNU // for O_PATH - #include #include #include diff --git a/tools/wasm2c/os.c b/tools/wasm2c/os.c index d029f8e393173..d73347fc61aba 100644 --- a/tools/wasm2c/os.c +++ b/tools/wasm2c/os.c @@ -235,7 +235,7 @@ IMPORT_IMPL(u32, Z_envZ___sys_fstat64Z_iii, (u32 fd, u32 buf), { IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), { VERBOSE_LOG(" stat64: %s\n", MEMACCESS(path)); - int nfd = open(MEMACCESS(path), O_PATH); + int nfd = open(MEMACCESS(path), O_RDONLY); // could be O_PATH on linux... if (nfd < 0) { VERBOSE_LOG(" error, %d %s\n", errno, strerror(errno)); return EM_EACCES; From d5307164d95bd9ef05ec4d66f9e71daeccad99a0 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 14:23:10 -0700 Subject: [PATCH 33/49] more [ci skip] --- tests/test_other.py | 2 ++ tools/shared.py | 9 +++++++++ tools/wasm2c/reactor.c | 15 +++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 tools/wasm2c/reactor.c diff --git a/tests/test_other.py b/tests/test_other.py index 6319e491614ec..200c77d45e0ed 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -10269,6 +10269,8 @@ def test_standalone_syscalls(self): for engine in WASM_ENGINES: self.assertContained(expected, run_js('test.wasm', engine)) +# TODO test wasm2c reactor + @no_fastcomp('wasm2js only') def test_promise_polyfill(self): def test(args): diff --git a/tools/shared.py b/tools/shared.py index e214f4e31fcb8..6abe07abb328c 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2824,6 +2824,15 @@ def do_wasm2c(infile): # an executable with main, as opposed to a library support_files.append('os') support_files.append('main') + else: + support_files.append('reactor') + # for a reactor, also append wasmbox_* API definitions + with open(h_file, 'a') as f: + f.write(''' +// wasmbox_* API +// TODO: optional prefixing +extern void wasmbox_init(void); +''') for support_file in support_files: with open(path_from_root('tools', 'wasm2c', support_file + '.c')) as f: total += f.read() diff --git a/tools/wasm2c/reactor.c b/tools/wasm2c/reactor.c new file mode 100644 index 0000000000000..565312b157e53 --- /dev/null +++ b/tools/wasm2c/reactor.c @@ -0,0 +1,15 @@ + +// TODO: optional prefixing +void wasmbox_init(void) { + // Initialize wasm2c runtime. + init(); + + // Set up handling for a trap + int trap_code; + if ((trap_code = setjmp(g_jmp_buf))) { + printf("[wasm trap %d, halting]\n", trap_code); + abort(); + } else { + Z__initializeZ_vv(); + } +} From 4f8e128ab38546ad3cbacb2bf71b3ef47dab2187 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 22 May 2020 15:37:00 -0700 Subject: [PATCH 34/49] nicer --- tools/shared.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 6abe07abb328c..5b12f09fbc9dd 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2799,6 +2799,12 @@ def do_wasm2c(infile): */ ''' SEP = '\n/* ==================================== */\n' + + def bundle_file(total, filename): + with open(filename) as f: + total += '// ' + filename + '\n' + f.read() + SEP + return total + # hermeticize the C file, by bundling in the wasm2c/ includes headers = [ (WASM2C_DIR, 'wasm-rt.h'), @@ -2806,15 +2812,13 @@ def do_wasm2c(infile): (os.path.dirname(h_file), os.path.basename(h_file)) ] for header in headers: - with open(os.path.join(header[0], header[1])) as f: - total += f.read() + SEP + total = bundle_file(total, os.path.join(header[0], header[1])) # add the wasm2c output with open(c_file) as read_c: c = read_c.read() total += c + SEP # add the wasm2c runtime - with open(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) as f: - total += f.read() + SEP + total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) # add the support code support_files = ['base'] if Settings.AUTODEBUG: @@ -2834,8 +2838,7 @@ def do_wasm2c(infile): extern void wasmbox_init(void); ''') for support_file in support_files: - with open(path_from_root('tools', 'wasm2c', support_file + '.c')) as f: - total += f.read() + total = bundle_file(total, path_from_root('tools', 'wasm2c', support_file + '.c')) # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) From 6059231876d6e6929b4b87eb0de3a0fb3a8dc867 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 06:23:49 -0700 Subject: [PATCH 35/49] simpler --- tools/wasm2c/base.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c index a2bc8f717ddfa..22b655de524ef 100644 --- a/tools/wasm2c/base.c +++ b/tools/wasm2c/base.c @@ -102,17 +102,6 @@ static jmp_buf setjmp_stack[MAX_SETJMP_STACK]; static u32 next_setjmp = 0; -// Declare exports for invokes. We should generate them based on what the -// wasm needs, but for now have a fixed list here. To get things to link, -// declare them, so they either link with the existing value in the main -// wasm2c .c output file, or else they contain NULL but will never be called. - -#define DECLARE_EXPORT(ret, name, args) \ -__attribute__((weak)) \ -ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; - -DECLARE_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); - // Stack support should be linked in if it is needed. IMPORT_IMPL(__attribute__((weak)) u32, Z_stackSaveZ_iv, (), { abort(); @@ -122,8 +111,6 @@ IMPORT_IMPL(__attribute__((weak))void, Z_stackRestoreZ_vi, (u32 x), { }); #define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ -DECLARE_EXPORT(void, dyncall, types); \ -\ IMPORT_IMPL(void, name, typed_args, { \ VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ u32 sp = Z_stackSaveZ_iv(); \ @@ -144,8 +131,6 @@ IMPORT_IMPL(void, name, typed_args, { \ }); #define RETURNING_INVOKE_IMPL(ret, name, typed_args, types, args, dyncall) \ -DECLARE_EXPORT(ret, dyncall, types); \ -\ IMPORT_IMPL(ret, name, typed_args, { \ VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ u32 sp = Z_stackSaveZ_iv(); \ @@ -167,6 +152,16 @@ IMPORT_IMPL(ret, name, typed_args, { \ return returned_value; \ }); +// Declare an export that may be needed and may not be. For example if longjmp +// is included then we need setThrew, but we must declare setThrew so that +// the C compiler can build us without error if longjmp is not actually used. + +#define DECLARE_WEAK_EXPORT(ret, name, args) \ +__attribute__((weak)) \ +ret (*WASM_RT_ADD_PREFIX(name)) args = NULL; + +DECLARE_WEAK_EXPORT(void, Z_setThrewZ_vii, (u32, u32)); + IMPORT_IMPL(void, Z_envZ_emscripten_longjmpZ_vii, (u32 buf, u32 value), { if (next_setjmp == 0) { abort_with_message("longjmp without setjmp"); From 50e7b18aa87cdb87a88307cbd74029fc86c27224 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 06:25:32 -0700 Subject: [PATCH 36/49] yet simpler --- tools/wasm2c/base.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c index 22b655de524ef..8ca43ab508297 100644 --- a/tools/wasm2c/base.c +++ b/tools/wasm2c/base.c @@ -79,7 +79,7 @@ DEFINE_STORE(wasm_i64_store32, u32, u64); #endif #define IMPORT_IMPL(ret, name, params, body) \ -ret _##name params { \ +static ret _##name params { \ VERBOSE_LOG("[import: " #name "]\n"); \ body \ } \ @@ -102,14 +102,6 @@ static jmp_buf setjmp_stack[MAX_SETJMP_STACK]; static u32 next_setjmp = 0; -// Stack support should be linked in if it is needed. -IMPORT_IMPL(__attribute__((weak)) u32, Z_stackSaveZ_iv, (), { - abort(); -}); -IMPORT_IMPL(__attribute__((weak))void, Z_stackRestoreZ_vi, (u32 x), { - abort(); -}); - #define VOID_INVOKE_IMPL(name, typed_args, types, args, dyncall) \ IMPORT_IMPL(void, name, typed_args, { \ VERBOSE_LOG("invoke " #name " " #dyncall "\n"); \ From 70dd79fe721ceb9274268e2e5459e6e099644ee0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 09:36:15 -0700 Subject: [PATCH 37/49] wip [ci skip] --- tests/test_other.py | 19 ++++++++++++++++++- tools/wasm2c/base.c | 31 +++++++++++++++++++++++++++++++ tools/wasm2c/os.c | 33 +-------------------------------- tools/wasm2c/os_sandboxed.c | 22 ++++++++++++++++++++++ 4 files changed, 72 insertions(+), 33 deletions(-) create mode 100644 tools/wasm2c/os_sandboxed.c diff --git a/tests/test_other.py b/tests/test_other.py index 200c77d45e0ed..3b9a511bf2ca0 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -10269,7 +10269,24 @@ def test_standalone_syscalls(self): for engine in WASM_ENGINES: self.assertContained(expected, run_js('test.wasm', engine)) -# TODO test wasm2c reactor + @no_fastcomp("uses standalone mode") + def test_wasm2c_reactor(self): + # test compiling an unsafe library using wasm2c, then using it from a + # main program. this shows it is easy to use wasm2c as a sandboxing + # mechanism. + + # first compile the library with emcc, getting a .c and .h + run_process([PYTHON, EMCC, + path_from_root('tests', 'other', 'wasm2c', 'unsafe-library.c'), + '-O3', '-o', 'lib.wasm', '-s', 'WASM2C', '--no-entry']) + # compile that .c to a native object + run_process([CLANG_CC, 'lib.wasm.c', '-c', '-O3', '-o', 'lib.o']) + # compile the main program natively normally, and link with the + # unsafe library + run_process([CLANG_CC, + path_from_root('tests', 'other', 'wasm2c', 'my-code.c'), + '-O3', 'lib.o', '-o', 'program.exe']) + run_process(['program.exe']) @no_fastcomp('wasm2js only') def test_promise_polyfill(self): diff --git a/tools/wasm2c/base.c b/tools/wasm2c/base.c index 8ca43ab508297..2d52dd32e2f1f 100644 --- a/tools/wasm2c/base.c +++ b/tools/wasm2c/base.c @@ -173,3 +173,34 @@ IMPORT_IMPL(u32, Z_envZ_getTempRet0Z_iv, (), { IMPORT_IMPL(void, Z_envZ_setTempRet0Z_vi, (u32 x), { tempRet0 = x; }); + +// Shared OS support in both sandboxed and unsandboxed mode + +#define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ +#define WASI_EINVAL 28 + +// Syscalls return a negative error code +#define EM_EACCES -2 + +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_fdstat_getZ_iii, (u32 a, u32 b), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_syncZ_ii, (u32 a), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_envZ_dlopenZ_iii, (u32 a, u32 b), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlcloseZ_ii, (u32 a), 1); +STUB_IMPORT_IMPL(u32, Z_envZ_dlsymZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_dlerrorZ_iv, (), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_signalZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_systemZ_ii, (u32 a), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_utimesZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_rmdirZ_ii, (u32 a), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_renameZ_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_lstat64Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup3Z_iiii, (u32 a, u32 b, u32 c), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup2Z_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_getcwdZ_iii, (u32 a, u32 b), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_ftruncate64Z_iiiii, (u32 a, u32 b, u32 c, u32 d), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_initZ_ii, (u32 a), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_settypeZ_iii, (u32 a, u32 b), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_destroyZ_ii, (u32 a), 0); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); +STUB_IMPORT_IMPL(u32, Z_envZ_pthread_joinZ_iii, (u32 a, u32 b), -1); +STUB_IMPORT_IMPL(u32, Z_envZ___cxa_thread_atexitZ_iiii, (u32 a, u32 b, u32 c), -1); diff --git a/tools/wasm2c/os.c b/tools/wasm2c/os.c index d73347fc61aba..43ad41a8717df 100644 --- a/tools/wasm2c/os.c +++ b/tools/wasm2c/os.c @@ -1,6 +1,3 @@ -#define WASI_DEFAULT_ERROR 63 /* __WASI_ERRNO_PERM */ -#define WASI_EINVAL 28 - IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { exit(x); }); @@ -164,21 +161,8 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), { return Z_wasi_snapshot_preview1Z_fd_seekZ_iijii(a, b + (((u64)c) << 32), d, e); }); -STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_fdstat_getZ_iii, (u32 a, u32 b), WASI_DEFAULT_ERROR); -STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_syncZ_ii, (u32 a), WASI_DEFAULT_ERROR); - -// TODO: set errno in wasm for everything - -STUB_IMPORT_IMPL(u32, Z_envZ_dlopenZ_iii, (u32 a, u32 b), 1); -STUB_IMPORT_IMPL(u32, Z_envZ_dlcloseZ_ii, (u32 a), 1); -STUB_IMPORT_IMPL(u32, Z_envZ_dlsymZ_iii, (u32 a, u32 b), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_dlerrorZ_iv, (), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_signalZ_iii, (u32 a, u32 b), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_systemZ_ii, (u32 a), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_utimesZ_iii, (u32 a, u32 b), -1); -// Syscalls return a negative error code -#define EM_EACCES -2 +// TODO: set errno in wasm for things that need it IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), { VERBOSE_LOG(" unlink %s\n", MEMACCESS(path)); @@ -188,12 +172,6 @@ IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), { } return 0; }); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_rmdirZ_ii, (u32 a), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_renameZ_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_lstat64Z_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup3Z_iiii, (u32 a, u32 b, u32 c), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_dup2Z_iii, (u32 a, u32 b), EM_EACCES); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_getcwdZ_iii, (u32 a, u32 b), EM_EACCES); static u32 do_stat(int nfd, u32 buf) { struct stat nbuf; @@ -243,7 +221,6 @@ IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), { return do_stat(nfd, buf); }); -STUB_IMPORT_IMPL(u32, Z_envZ___sys_ftruncate64Z_iiiii, (u32 a, u32 b, u32 c, u32 d), EM_EACCES); IMPORT_IMPL(u32, Z_envZ___sys_readZ_iiii, (u32 fd, u32 buf, u32 count), { int nfd = get_native_fd(fd); VERBOSE_LOG(" read %d (=> %d) %d %d\n", fd, nfd, buf, count); @@ -271,10 +248,6 @@ IMPORT_IMPL(u32, Z_envZ___sys_accessZ_iii, (u32 pathname, u32 mode), { return 0; }); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_initZ_ii, (u32 a), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_settypeZ_iii, (u32 a, u32 b), 0); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_mutexattr_destroyZ_ii, (u32 a), 0); - #define WASM_CLOCK_REALTIME 0 #define WASM_CLOCK_MONOTONIC 1 #define WASM_CLOCK_PROCESS_CPUTIME 2 @@ -305,7 +278,3 @@ IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u3 wasm_i64_store(out, 1000 * 1000); return 0; }); - -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_createZ_iiiii, (u32 a, u32 b, u32 c, u32 d), -1); -STUB_IMPORT_IMPL(u32, Z_envZ_pthread_joinZ_iii, (u32 a, u32 b), -1); -STUB_IMPORT_IMPL(u32, Z_envZ___cxa_thread_atexitZ_iiii, (u32 a, u32 b, u32 c), -1); diff --git a/tools/wasm2c/os_sandboxed.c b/tools/wasm2c/os_sandboxed.c new file mode 100644 index 0000000000000..728d7130cb006 --- /dev/null +++ b/tools/wasm2c/os_sandboxed.c @@ -0,0 +1,22 @@ +// Stubs for OS functions, for a sandboxed environment. Nothing is allowed +// exit the sandbox, calls to printf will fail, etc. + +IMPORT_IMPL(void, Z_wasi_snapshot_preview1Z_proc_exitZ_vi, (u32 x), { + abort_with_message("exit() called"); +}); + +STUB_IMPORT_IMPL(u32, Z_envZ___sys_openZ_iiii, (u32 path, u32 flags, u32 varargs), -1); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_writeZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_readZ_iiiii, (u32 fd, u32 iov, u32 iovcnt, u32 pnum), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_closeZ_ii, (u32 fd), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_sizes_getZ_iii, (u32 pcount, u32 pbuf_size), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_environ_getZ_iii, (u32 __environ, u32 environ_buf), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iijii, (u32 fd, u64 offset, u32 whence, u32 new_offset), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_fd_seekZ_iiiiii, (u32 a, u32 b, u32 c, u32 d, u32 e), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_unlinkZ_ii, (u32 path), WASI_DEFAULT_ERROR); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_fstat64Z_iii, (u32 fd, u32 buf), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_stat64Z_iii, (u32 path, u32 buf), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_readZ_iiii, (u32 fd, u32 buf, u32 count), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_envZ___sys_accessZ_iii, (u32 pathname, u32 mode), EM_EACCES); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_time_getZ_iiji, (u32 clock_id, u64 max_lag, u32 out), WASI_EINVAL); +STUB_IMPORT_IMPL(u32, Z_wasi_snapshot_preview1Z_clock_res_getZ_iii, (u32 clock_id, u32 out), WASI_EINVAL); From 35ffd9790f16bb7fad43c698d144a72d4b036eb3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 09:41:48 -0700 Subject: [PATCH 38/49] finish test --- tests/other/wasm2c/my-code.c | 20 ++++++++++++++++++++ tests/other/wasm2c/output.txt | 5 +++++ tests/other/wasm2c/unsafe-library.c | 15 +++++++++++++++ tests/test_other.py | 4 +++- tools/shared.py | 1 + 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/other/wasm2c/my-code.c create mode 100644 tests/other/wasm2c/output.txt create mode 100644 tests/other/wasm2c/unsafe-library.c diff --git a/tests/other/wasm2c/my-code.c b/tests/other/wasm2c/my-code.c new file mode 100644 index 0000000000000..63f75f3a2cec5 --- /dev/null +++ b/tests/other/wasm2c/my-code.c @@ -0,0 +1,20 @@ +#include + +// We could also include the .wasm.h file for +// these, but let's declare things for the example. + +extern void wasmbox_init(void); + +extern int (*Z_do_bad_thingZ_ii)(int); + +extern int (*Z_twiceZ_ii)(int); + +int main() { + puts("Initializing sandboxed unsafe library"); + wasmbox_init(); + printf("Calling twice on 21 returns %d\n", Z_twiceZ_ii(21)); + puts("Calling something bad now..."); + int num = Z_do_bad_thingZ_ii(1); + printf("The sandbox should not have been able to print anything.\n" + "It claims it printed %d chars but the test proves it didn't!\n", num); +} diff --git a/tests/other/wasm2c/output.txt b/tests/other/wasm2c/output.txt new file mode 100644 index 0000000000000..5c0c8b30b7cb9 --- /dev/null +++ b/tests/other/wasm2c/output.txt @@ -0,0 +1,5 @@ +Initializing sandboxed unsafe library +Calling twice on 21 returns 42 +Calling something bad now... +The sandbox should not have been able to print anything. +It claims it printed 55 chars but the test proves it didn't! diff --git a/tests/other/wasm2c/unsafe-library.c b/tests/other/wasm2c/unsafe-library.c new file mode 100644 index 0000000000000..ac1076d2d9384 --- /dev/null +++ b/tests/other/wasm2c/unsafe-library.c @@ -0,0 +1,15 @@ +// unsafe-lib.c + +#include +#include + +EMSCRIPTEN_KEEPALIVE +int twice(int x) { + return x + x; +} + +EMSCRIPTEN_KEEPALIVE +int do_bad_thing(int size) { + return printf("I am in a sandbox and should not be able to print this!"); +} + diff --git a/tests/test_other.py b/tests/test_other.py index 3b9a511bf2ca0..693bb23c0f243 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -10286,7 +10286,9 @@ def test_wasm2c_reactor(self): run_process([CLANG_CC, path_from_root('tests', 'other', 'wasm2c', 'my-code.c'), '-O3', 'lib.o', '-o', 'program.exe']) - run_process(['program.exe']) + output = run_process([os.path.abspath('program.exe')], stdout=PIPE).stdout + with open(path_from_root('tests', 'other', 'wasm2c', 'output.txt')) as f: + self.assertEqual(output, f.read()) @no_fastcomp('wasm2js only') def test_promise_polyfill(self): diff --git a/tools/shared.py b/tools/shared.py index 5b12f09fbc9dd..f3164cfe7fbd8 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2829,6 +2829,7 @@ def bundle_file(total, filename): support_files.append('os') support_files.append('main') else: + support_files.append('os_sandboxed') support_files.append('reactor') # for a reactor, also append wasmbox_* API definitions with open(h_file, 'a') as f: From 56ffcf3eb6a07849ba5c0fb644b7f8fa4ea7d821 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 10:33:15 -0700 Subject: [PATCH 39/49] more [ci skip] --- tests/other/wasm2c/my-code.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/other/wasm2c/my-code.c b/tests/other/wasm2c/my-code.c index 63f75f3a2cec5..ab263ab35405d 100644 --- a/tests/other/wasm2c/my-code.c +++ b/tests/other/wasm2c/my-code.c @@ -1,7 +1,12 @@ #include -// We could also include the .wasm.h file for -// these, but let's declare things for the example. +// We could +// +// #include +// +// for the externs declared here manually, but including that currently +// requires having wasm-rt.h in the include path, which may be annoying for +// users - needs to be thought about. extern void wasmbox_init(void); From d490cf9308735251ec88f881f762440a63806e4a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 12:09:50 -0700 Subject: [PATCH 40/49] flake8 --- tests/test_benchmark.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index c5f1006137004..29f70dadb46c3 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -8,7 +8,6 @@ import os import re import shutil -import subprocess import sys import time import unittest @@ -275,7 +274,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat self.filename = native def run(self, args): - return run_process([self.filename] + args, stdout=PIPE, stderr=subprocess.STDOUT, check=False).stdout + return run_process([self.filename] + args, stdout=PIPE, stderr=STDOUT, check=False).stdout def get_output_files(self): # return the native code. c size may also be interesting. From 73924dd68436fa62a0cdca31d217c3e8d8737ea4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 23 May 2020 12:12:38 -0700 Subject: [PATCH 41/49] flake8 --- tests/test_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 29f70dadb46c3..5cbbf7dc56346 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -274,7 +274,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat self.filename = native def run(self, args): - return run_process([self.filename] + args, stdout=PIPE, stderr=STDOUT, check=False).stdout + return run_process([self.filename] + args, stdout=PIPE, stderr=PIPE, check=False).stdout def get_output_files(self): # return the native code. c size may also be interesting. From cd1bfbf07bfa8309f849ea5e6267b3b6819f74f9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sun, 24 May 2020 14:10:08 -0700 Subject: [PATCH 42/49] separate out --- emcc.py | 4 +- tools/shared.py | 102 ------------------------------------ tools/wasm2c/__init__.py | 110 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 103 deletions(-) create mode 100644 tools/wasm2c/__init__.py diff --git a/emcc.py b/emcc.py index 42d9d1bb4563c..3ef41fc7a2053 100755 --- a/emcc.py +++ b/emcc.py @@ -46,6 +46,8 @@ from tools.minimal_runtime_shell import generate_minimal_runtime_html import tools.line_endings from tools.toolchain_profiler import ToolchainProfiler +from tools import wasm2c + if __name__ == '__main__': ToolchainProfiler.record_process_start() @@ -3342,7 +3344,7 @@ def run_closure_compiler(final): shared.Building.emit_debug_on_side(wasm_binary_target, dwarf_target) if shared.Settings.WASM2C: - shared.Building.do_wasm2c(wasm_binary_target) + wasm2c.do_wasm2c(wasm_binary_target) # replace placeholder strings with correct subresource locations if shared.Settings.SINGLE_FILE: diff --git a/tools/shared.py b/tools/shared.py index 5f81acd460c32..527b989fbaa98 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2781,108 +2781,6 @@ def run_binaryen_command(tool, infile, outfile=None, args=[], debug=False, stdou def run_wasm_opt(*args, **kwargs): return Building.run_binaryen_command('wasm-opt', *args, **kwargs) - @staticmethod - def do_wasm2c(infile): - assert Settings.STANDALONE_WASM - WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] - WASM2C_DIR = path_from_root('node_modules', 'wasm2c') - c_file = unsuffixed(infile) + '.wasm.c' - h_file = unsuffixed(infile) + '.wasm.h' - cmd = WASM2C + [infile, '-o', c_file] - run_process(cmd) - total = '''\ -/* - * This file was generated by emcc+wasm2c. To compile it, use something like - * - * $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 - */ -''' - SEP = '\n/* ==================================== */\n' - - def bundle_file(total, filename): - with open(filename) as f: - total += '// ' + filename + '\n' + f.read() + SEP - return total - - # hermeticize the C file, by bundling in the wasm2c/ includes - headers = [ - (WASM2C_DIR, 'wasm-rt.h'), - (WASM2C_DIR, 'wasm-rt-impl.h'), - (os.path.dirname(h_file), os.path.basename(h_file)) - ] - for header in headers: - total = bundle_file(total, os.path.join(header[0], header[1])) - # add the wasm2c output - with open(c_file) as read_c: - c = read_c.read() - total += c + SEP - # add the wasm2c runtime - total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) - # add the support code - support_files = ['base'] - if Settings.AUTODEBUG: - support_files.append('autodebug') - if Settings.EXPECT_MAIN: - # TODO: add an option for direct OS access. For now, do that when building - # an executable with main, as opposed to a library - support_files.append('os') - support_files.append('main') - else: - support_files.append('os_sandboxed') - support_files.append('reactor') - # for a reactor, also append wasmbox_* API definitions - with open(h_file, 'a') as f: - f.write(''' -// wasmbox_* API -// TODO: optional prefixing -extern void wasmbox_init(void); -''') - for support_file in support_files: - total = bundle_file(total, path_from_root('tools', 'wasm2c', support_file + '.c')) - # remove #includes of the headers we bundled - for header in headers: - total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) - # generate the necessary invokes - invokes = [] - for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): - def s_to_c(s): - if s == 'v': - return 'void' - elif s == 'i': - return 'u32' - elif s == 'j': - return 'u64' - elif s == 'f': - return 'f32' - elif s == 'd': - return 'f64' - else: - exit_with_error('invalid sig element:' + str(s)) - - def name(i): - return 'a' + str(i) - - wabt_sig = sig[0] + 'i' + sig[1:] - typed_args = ['u32 fptr'] + [s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))] - types = ['u32'] + [s_to_c(sig[i]) for i in range(1, len(sig))] - args = ['fptr'] + [name(i) for i in range(1, len(sig))] - invokes.append( - '%s_INVOKE_IMPL(%sZ_envZ_invoke_%sZ_%s, (%s), (%s), (%s), Z_dynCall_%sZ_%s);' % ( - 'VOID' if sig[0] == 'v' else 'RETURNING', - (s_to_c(sig[0]) + ', ') if sig[0] != 'v' else '', - sig, - wabt_sig, - ', '.join(typed_args), - ', '.join(types), - ', '.join(args), - sig, - wabt_sig - )) - total += '\n'.join(invokes) - # write out the final file - with open(c_file, 'w') as out: - out.write(total) - save_intermediate_counter = 0 @staticmethod diff --git a/tools/wasm2c/__init__.py b/tools/wasm2c/__init__.py new file mode 100644 index 0000000000000..4085d3d143af8 --- /dev/null +++ b/tools/wasm2c/__init__.py @@ -0,0 +1,110 @@ +# Copyright 2020 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import re + +from tools.shared import Settings, path_from_root, unsuffixed, NODE_JS, run_process + +def do_wasm2c(infile): + assert Settings.STANDALONE_WASM + WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] + WASM2C_DIR = path_from_root('node_modules', 'wasm2c') + c_file = unsuffixed(infile) + '.wasm.c' + h_file = unsuffixed(infile) + '.wasm.h' + cmd = WASM2C + [infile, '-o', c_file] + run_process(cmd) + total = '''\ +/* +* This file was generated by emcc+wasm2c. To compile it, use something like +* +* $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 +*/ +''' + SEP = '\n/* ==================================== */\n' + + def bundle_file(total, filename): + with open(filename) as f: + total += '// ' + filename + '\n' + f.read() + SEP + return total + + # hermeticize the C file, by bundling in the wasm2c/ includes + headers = [ + (WASM2C_DIR, 'wasm-rt.h'), + (WASM2C_DIR, 'wasm-rt-impl.h'), + (os.path.dirname(h_file), os.path.basename(h_file)) + ] + for header in headers: + total = bundle_file(total, os.path.join(header[0], header[1])) + # add the wasm2c output + with open(c_file) as read_c: + c = read_c.read() + total += c + SEP + # add the wasm2c runtime + total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) + # add the support code + support_files = ['base'] + if Settings.AUTODEBUG: + support_files.append('autodebug') + if Settings.EXPECT_MAIN: + # TODO: add an option for direct OS access. For now, do that when building + # an executable with main, as opposed to a library + support_files.append('os') + support_files.append('main') + else: + support_files.append('os_sandboxed') + support_files.append('reactor') + # for a reactor, also append wasmbox_* API definitions + with open(h_file, 'a') as f: + f.write(''' +// wasmbox_* API +// TODO: optional prefixing +extern void wasmbox_init(void); +''') + for support_file in support_files: + total = bundle_file(total, path_from_root('tools', 'wasm2c', support_file + '.c')) + # remove #includes of the headers we bundled + for header in headers: + total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) + # generate the necessary invokes + invokes = [] + for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): + def s_to_c(s): + if s == 'v': + return 'void' + elif s == 'i': + return 'u32' + elif s == 'j': + return 'u64' + elif s == 'f': + return 'f32' + elif s == 'd': + return 'f64' + else: + exit_with_error('invalid sig element:' + str(s)) + + def name(i): + return 'a' + str(i) + + wabt_sig = sig[0] + 'i' + sig[1:] + typed_args = ['u32 fptr'] + [s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))] + types = ['u32'] + [s_to_c(sig[i]) for i in range(1, len(sig))] + args = ['fptr'] + [name(i) for i in range(1, len(sig))] + invokes.append( + '%s_INVOKE_IMPL(%sZ_envZ_invoke_%sZ_%s, (%s), (%s), (%s), Z_dynCall_%sZ_%s);' % ( + 'VOID' if sig[0] == 'v' else 'RETURNING', + (s_to_c(sig[0]) + ', ') if sig[0] != 'v' else '', + sig, + wabt_sig, + ', '.join(typed_args), + ', '.join(types), + ', '.join(args), + sig, + wabt_sig + )) + total += '\n'.join(invokes) + # write out the final file + with open(c_file, 'w') as out: + out.write(total) From 83502a564c1c7f84b9330ca004bd149c647e1276 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sun, 24 May 2020 18:02:58 -0700 Subject: [PATCH 43/49] flake8 --- tools/wasm2c/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/wasm2c/__init__.py b/tools/wasm2c/__init__.py index 4085d3d143af8..1452c826d4e0d 100644 --- a/tools/wasm2c/__init__.py +++ b/tools/wasm2c/__init__.py @@ -6,7 +6,8 @@ import os import re -from tools.shared import Settings, path_from_root, unsuffixed, NODE_JS, run_process +from tools.shared import Settings, path_from_root, unsuffixed, NODE_JS, run_process, exit_with_error + def do_wasm2c(infile): assert Settings.STANDALONE_WASM From 5f3abb917012e51c61a50297cb5946afc48ef7d4 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Mon, 25 May 2020 16:13:05 -0700 Subject: [PATCH 44/49] rename --- tools/{wasm2c/__init__.py => wasm2c.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{wasm2c/__init__.py => wasm2c.py} (100%) diff --git a/tools/wasm2c/__init__.py b/tools/wasm2c.py similarity index 100% rename from tools/wasm2c/__init__.py rename to tools/wasm2c.py From 33f804accd426743cb68ac8954c104c0f18aa6da Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 28 May 2020 13:15:16 -0700 Subject: [PATCH 45/49] simplify --- tests/test_core.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 3b0a7976dc2e0..f7fadc2dbaa4d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -249,24 +249,6 @@ def decorated(self): return decorated -def also_with_only_standalone_wasm_and_wasm2c(func): - def decorated(self): - func(self) - # Standalone mode is only supported in the wasm backend, and not in all - # modes there. - if can_do_standalone(self): - with js_engines_modify([]): - print('standalone (only; no js runtimes)') - self.set_setting('STANDALONE_WASM', 1) - func(self) - print('wasm2c') - self.set_setting('STANDALONE_WASM', 1) - self.set_setting('WASM2C', 1) - with wasm_engines_modify([]): - func(self) - return decorated - - def node_pthreads(f): def decorated(self): self.set_setting('USE_PTHREADS', 1) From 4028cab2b0e5f934c05db658e4ee32bda5bd4079 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 4 Jun 2020 16:13:26 -0700 Subject: [PATCH 46/49] fix after merge --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 70b02b83caafc..af5dae51a888b 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -10255,7 +10255,7 @@ def test_wasm2c_reactor(self): # mechanism. # first compile the library with emcc, getting a .c and .h - run_process([PYTHON, EMCC, + run_process([EMCC, path_from_root('tests', 'other', 'wasm2c', 'unsafe-library.c'), '-O3', '-o', 'lib.wasm', '-s', 'WASM2C', '--no-entry']) # compile that .c to a native object From fe058197b90940ccde11d7ba205f5b85c20419a8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 4 Jun 2020 19:21:33 -0700 Subject: [PATCH 47/49] fix --- tools/wasm2c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/wasm2c.py b/tools/wasm2c.py index 1452c826d4e0d..5b0a3c6cf63e5 100644 --- a/tools/wasm2c.py +++ b/tools/wasm2c.py @@ -35,7 +35,7 @@ def bundle_file(total, filename): headers = [ (WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'), - (os.path.dirname(h_file), os.path.basename(h_file)) + ('', h_file) ] for header in headers: total = bundle_file(total, os.path.join(header[0], header[1])) From 5f99dc50dc849bf694da117b2a5cc6eef7977736 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 5 Jun 2020 14:41:35 -0700 Subject: [PATCH 48/49] cleanup --- tests/other/wasm2c/unsafe-library.c | 3 --- tools/wasm2c/reactor.c | 1 - 2 files changed, 4 deletions(-) diff --git a/tests/other/wasm2c/unsafe-library.c b/tests/other/wasm2c/unsafe-library.c index ac1076d2d9384..28d4a85783180 100644 --- a/tests/other/wasm2c/unsafe-library.c +++ b/tests/other/wasm2c/unsafe-library.c @@ -1,5 +1,3 @@ -// unsafe-lib.c - #include #include @@ -12,4 +10,3 @@ EMSCRIPTEN_KEEPALIVE int do_bad_thing(int size) { return printf("I am in a sandbox and should not be able to print this!"); } - diff --git a/tools/wasm2c/reactor.c b/tools/wasm2c/reactor.c index 565312b157e53..134f1a6a3eff2 100644 --- a/tools/wasm2c/reactor.c +++ b/tools/wasm2c/reactor.c @@ -1,4 +1,3 @@ - // TODO: optional prefixing void wasmbox_init(void) { // Initialize wasm2c runtime. From 8dbc4d44d1489faa166b2099b5a6de2012e19396 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 5 Jun 2020 15:40:17 -0700 Subject: [PATCH 49/49] restore test --- tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_core.py b/tests/test_core.py index 430c058f790aa..b7f66b69e95ff 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -392,6 +392,7 @@ def get_bullet_library(self, use_cmake): configure_args=configure_args, cache_name_extra=configure_commands[0]) + @also_with_standalone_wasm def test_hello_world(self): self.do_run_in_out_file_test('tests', 'core', 'test_hello_world') # must not emit this unneeded internal thing