From d129523c4d396f20f070fe221cca08ad49fc78e6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Mar 2024 07:37:20 -0700 Subject: [PATCH 1/8] Update LLVM to 18.1.2 and add wasm-component-ld This commit has two intertwined changes within it. First the LLVM submodule is updated to the 18.1.2 release branch. This alone would cause CI and tests to fail due to differing behavior for the `wasm32-wasip2` target. To fix these test failures the `wasm-component-ld` tool is added to the build. This tool is a Rust-written tool and installed via `cargo install` as part of the build at a pinned version written in the `Makefile`. This linker, used for components, is then used for the `wasm32-wasip2` target. Tests and CI are then updated to skip the need to have `wasm-tools` or the adapter for WASI when making components since that's now the job of `wasm-component-ld`. This then necessitated some changes to tests too. --- .github/workflows/main.yml | 11 +------- Makefile | 9 +++++-- src/llvm-project | 2 +- tests/general/abort.c.stderr.expected | 2 +- tests/general/abort.c.stderr.expected.filter | 1 + tests/general/assert-fail.c.stderr.expected | 2 +- .../assert-fail.c.stderr.expected.filter | 1 + tests/general/sigabrt.c.stderr.expected | 2 +- .../general/sigabrt.c.stderr.expected.filter | 1 + tests/run.sh | 25 ++++++------------- tests/testcase.sh | 16 +++--------- 11 files changed, 27 insertions(+), 45 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4303d08d6..b167bebf0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,15 +42,6 @@ jobs: uses: bytecodealliance/actions/wasmtime/setup@v1 with: version: "18.0.2" - - name: Setup `wasm-tools` for tests - uses: bytecodealliance/actions/wasm-tools/setup@v1 - with: - version: "1.201.0" - - name: Download command adapter for tests - run: curl -f -L --retry 5 -o ${{ runner.temp }}/wasi_snapshot_preview1.command.wasm https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.command.wasm - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # We can't use `--depth 1` here sadly because the GNU config # submodule is not pinned to a particular tag/branch. Please # bump depth (or even better, the submodule), in case of "error: @@ -67,7 +58,7 @@ jobs: run: NINJA_FLAGS=-v make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON shell: bash - name: Run the testsuite - run: NINJA_FLAGS=-v make check RUNTIME=wasmtime ADAPTER=${{ runner.temp }}/wasi_snapshot_preview1.command.wasm WASM_TOOLS=wasm-tools + run: NINJA_FLAGS=-v make check RUNTIME=wasmtime - name: Upload artifacts uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 9c7acffbf..e980abcc3 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ default: build @echo "Use -fdebug-prefix-map=$(ROOT_DIR)=wasisdk://v$(VERSION)" check: - TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)" "$(ADAPTER)" "$(WASM_TOOLS)" + TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)" clean: rm -rf build $(DESTDIR) @@ -122,6 +122,11 @@ build/llvm.BUILT: llvm-config touch build/llvm.BUILT +build/wasm-component-ld.BUILT: build/llvm.BUILT + cargo install wasm-component-ld@0.1.5 --root $(BUILD_PREFIX) + touch build/wasm-component-ld.BUILT + + # Flags for running `make` in wasi-libc # $(1): the target that's being built WASI_LIBC_MAKEFLAGS = \ @@ -132,7 +137,7 @@ WASI_LIBC_MAKEFLAGS = \ SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ TARGET_TRIPLE=$(1) -build/wasi-libc.BUILT: build/compiler-rt.BUILT +build/wasi-libc.BUILT: build/compiler-rt.BUILT build/wasm-component-ld.BUILT $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasi) default libc_so $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip1) default libc_so $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip2) WASI_SNAPSHOT=p2 default libc_so diff --git a/src/llvm-project b/src/llvm-project index 461274b81..26a1d6601 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 461274b81d8641eab64d494accddc81d7db8a09e +Subproject commit 26a1d6601d727a96f4301d0d8647b5a42760ae0c diff --git a/tests/general/abort.c.stderr.expected b/tests/general/abort.c.stderr.expected index c6ecf6a93..94e6cd1ff 100644 --- a/tests/general/abort.c.stderr.expected +++ b/tests/general/abort.c.stderr.expected @@ -1,6 +1,6 @@ Error: failed to run main module `abort.c.---.wasm` Caused by: - 0: failed to invoke command default + 0: failed to invoke 1: error while executing at wasm backtrace: 2: wasm trap: wasm `unreachable` instruction executed diff --git a/tests/general/abort.c.stderr.expected.filter b/tests/general/abort.c.stderr.expected.filter index 742503a5c..272cd59ad 100755 --- a/tests/general/abort.c.stderr.expected.filter +++ b/tests/general/abort.c.stderr.expected.filter @@ -3,4 +3,5 @@ set -euo pipefail cat \ | sed -e 's/main module `abort\.c\.[^`]*\.wasm`/main module `abort.c.---.wasm`/' \ + | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -E '/0x[[:xdigit:]]+/d' diff --git a/tests/general/assert-fail.c.stderr.expected b/tests/general/assert-fail.c.stderr.expected index c179f09ad..1579f8cfa 100644 --- a/tests/general/assert-fail.c.stderr.expected +++ b/tests/general/assert-fail.c.stderr.expected @@ -2,6 +2,6 @@ Assertion failed: false (assert-fail.c: main: 5) Error: failed to run main module `assert-fail.c.---.wasm` Caused by: - 0: failed to invoke command default + 0: failed to invoke 1: error while executing at wasm backtrace: 2: wasm trap: wasm `unreachable` instruction executed diff --git a/tests/general/assert-fail.c.stderr.expected.filter b/tests/general/assert-fail.c.stderr.expected.filter index d2213dd6b..f9ffe8460 100755 --- a/tests/general/assert-fail.c.stderr.expected.filter +++ b/tests/general/assert-fail.c.stderr.expected.filter @@ -3,4 +3,5 @@ set -euo pipefail cat \ | sed -e 's/main module `assert-fail\.c\.[^`]*\.wasm`/main module `assert-fail.c.---.wasm`/' \ + | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -E '/0x[[:xdigit:]]+/d' diff --git a/tests/general/sigabrt.c.stderr.expected b/tests/general/sigabrt.c.stderr.expected index 72c425d0c..605ac5db8 100644 --- a/tests/general/sigabrt.c.stderr.expected +++ b/tests/general/sigabrt.c.stderr.expected @@ -3,4 +3,4 @@ Program received fatal signal: Aborted Error: failed to run main module `sigabrt.c.---.wasm` Caused by: - 0: failed to invoke command default + 0: failed to invoke diff --git a/tests/general/sigabrt.c.stderr.expected.filter b/tests/general/sigabrt.c.stderr.expected.filter index 425b060c2..0f3d78f93 100755 --- a/tests/general/sigabrt.c.stderr.expected.filter +++ b/tests/general/sigabrt.c.stderr.expected.filter @@ -3,5 +3,6 @@ set -euo pipefail cat \ | sed -e 's/main module `sigabrt\.c\.[^`]*\.wasm`/main module `sigabrt.c.---.wasm`/' \ + | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -e 's/source location: @[[:xdigit:]]*$/source location: @----/' \ | head -n 6 diff --git a/tests/run.sh b/tests/run.sh index 080eeaee5..bc8dd4d0b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -17,17 +17,8 @@ wasi_sdk="$1" # Determine the wasm runtime to use, if one is provided. if [ $# -gt 1 ]; then runwasi="$2" - if [ $# -gt 3 ]; then - adapter="$3" - wasm_tools="$4" - else - adapter="" - wasm_tools="" - fi else runwasi="" - adapter="" - wasm_tools="" fi testdir=$(dirname $0) @@ -45,13 +36,13 @@ for target in $TARGETS; do echo "===== Testing compile-only with $options =====" for file in *.c; do echo "Testing compile-only $file..." - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang" "$options" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang" "$options" "$file" done for file in *.cc; do echo "Testing compile-only $file..." - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" done done cd - >/dev/null @@ -61,13 +52,13 @@ for target in $TARGETS; do echo "===== Testing with $options =====" for file in *.c; do echo "Testing $file..." - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang" "$options" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang" "$options" "$file" done for file in *.cc; do echo "Testing $file..." - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" done done cd - >/dev/null diff --git a/tests/testcase.sh b/tests/testcase.sh index 61c6612e5..3d8860464 100755 --- a/tests/testcase.sh +++ b/tests/testcase.sh @@ -8,18 +8,15 @@ set -ueo pipefail # script, so don't do anything fancy. target="$1" runwasi="$2" -adapter="$3" -wasm_tools="$4" -compiler="$5" -options="$6" -input="$7" +compiler="$3" +options="$4" +input="$5" # Compile names for generated files. wasm="$input.$options.wasm" stdout_observed="$input.$options.stdout.observed" stderr_observed="$input.$options.stderr.observed" exit_status_observed="$input.$options.exit_status.observed" -run_args="" # Optionally load compiler options from a file. if [ -e "$input.options" ]; then @@ -44,11 +41,6 @@ if [ "$runwasi" == "" ]; then exit 0 fi -if [ "$target" == "wasm32-wasip2" -a -n "$adapter" -a -n "$wasm_tools" ]; then - "$wasm_tools" component new --adapt "$adapter" "$wasm" -o "$wasm" - run_args="--wasm component-model" -fi - # Determine the input file to write to stdin. if [ -e "$input.stdin" ]; then stdin="$input.stdin" @@ -74,7 +66,7 @@ fi # Run the test, capturing stdout, stderr, and the exit status. exit_status=0 -"$runwasi" $run_args $env $dir "$wasm" $dirarg \ +"$runwasi" $env $dir "$wasm" $dirarg \ < "$stdin" \ > "$stdout_observed" \ 2> "$stderr_observed" \ From d744d47f3183943660892ae95ce4a75d3cf56a8d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Mar 2024 09:18:11 -0700 Subject: [PATCH 2/8] Add back accidentally deleted CI --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b167bebf0..3fda897a4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,6 +42,9 @@ jobs: uses: bytecodealliance/actions/wasmtime/setup@v1 with: version: "18.0.2" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # We can't use `--depth 1` here sadly because the GNU config # submodule is not pinned to a particular tag/branch. Please # bump depth (or even better, the submodule), in case of "error: From 7771f3141bdf9c3a4bd301c70f38e06897563f72 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Mar 2024 16:29:42 -0700 Subject: [PATCH 3/8] Inherit tools on PATH on Windows --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3fda897a4..0a2d00398 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,6 +101,7 @@ jobs: msystem: ${{ matrix.sys }} update: true release: false + path-type: inherit - uses: actions/checkout@v4 with: fetch-depth: 0 From 25796c942e01cc8dee00f672a4d05b8f1007ffbd Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Mar 2024 16:32:50 -0700 Subject: [PATCH 4/8] Add Rust to docker build --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9a864e63e..f03604bfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,9 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake- && mv cmake-3.25.1-linux-x86_64 /opt/cmake ENV PATH /opt/cmake/bin:$PATH +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal +ENV PATH /root/.cargo/bin:$PATH + RUN groupadd -g ${GID} builder && \ useradd --create-home --uid ${UID} --gid ${GID} builder USER builder From 082309f33851a38c7e1658967c00268b0722e1ab Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 31 Mar 2024 11:50:48 -0700 Subject: [PATCH 5/8] Put rust in a different location in docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f03604bfb..ca61db00b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,8 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake- && mv cmake-3.25.1-linux-x86_64 /opt/cmake ENV PATH /opt/cmake/bin:$PATH +ENV RUSTUP_HOME=/rust/rustup CARGO_HOME=/rust/cargo PATH=$PATH:/rust/cargo/bin RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal -ENV PATH /root/.cargo/bin:$PATH RUN groupadd -g ${GID} builder && \ useradd --create-home --uid ${UID} --gid ${GID} builder From 0e0079cfd880c7e44ac6a197e5110ef1f483f5ec Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 31 Mar 2024 13:41:46 -0700 Subject: [PATCH 6/8] Try to fix permissions --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ca61db00b..adddd029c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,9 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake- ENV PATH /opt/cmake/bin:$PATH ENV RUSTUP_HOME=/rust/rustup CARGO_HOME=/rust/cargo PATH=$PATH:/rust/cargo/bin -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile=minimal && \ + chmod -R a+w /rust RUN groupadd -g ${GID} builder && \ useradd --create-home --uid ${UID} --gid ${GID} builder From e98143ea7e452f8002a12c64cd2dab79a568aa8e Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 3 Apr 2024 07:43:43 -0700 Subject: [PATCH 7/8] Review comments --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index e980abcc3..fbe6af583 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,9 @@ build/llvm.BUILT: llvm-config touch build/llvm.BUILT +# Build the `wasm-component-ld` linker from source via `cargo install`. This is +# used for the `wasm32-wasip2` target natively by Clang. Note that `--root` +# passed to `cargo install` will place it in the output directory automatically. build/wasm-component-ld.BUILT: build/llvm.BUILT cargo install wasm-component-ld@0.1.5 --root $(BUILD_PREFIX) touch build/wasm-component-ld.BUILT From da880d821c6b4b90f586fe5f4ff507f61853282c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 3 Apr 2024 10:26:52 -0700 Subject: [PATCH 8/8] Revert changes to test outputs --- tests/general/abort.c.stderr.expected | 2 +- tests/general/abort.c.stderr.expected.filter | 1 - tests/general/assert-fail.c.stderr.expected | 2 +- tests/general/assert-fail.c.stderr.expected.filter | 1 - tests/general/sigabrt.c.stderr.expected | 2 +- tests/general/sigabrt.c.stderr.expected.filter | 1 - 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/general/abort.c.stderr.expected b/tests/general/abort.c.stderr.expected index 94e6cd1ff..c6ecf6a93 100644 --- a/tests/general/abort.c.stderr.expected +++ b/tests/general/abort.c.stderr.expected @@ -1,6 +1,6 @@ Error: failed to run main module `abort.c.---.wasm` Caused by: - 0: failed to invoke + 0: failed to invoke command default 1: error while executing at wasm backtrace: 2: wasm trap: wasm `unreachable` instruction executed diff --git a/tests/general/abort.c.stderr.expected.filter b/tests/general/abort.c.stderr.expected.filter index 272cd59ad..742503a5c 100755 --- a/tests/general/abort.c.stderr.expected.filter +++ b/tests/general/abort.c.stderr.expected.filter @@ -3,5 +3,4 @@ set -euo pipefail cat \ | sed -e 's/main module `abort\.c\.[^`]*\.wasm`/main module `abort.c.---.wasm`/' \ - | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -E '/0x[[:xdigit:]]+/d' diff --git a/tests/general/assert-fail.c.stderr.expected b/tests/general/assert-fail.c.stderr.expected index 1579f8cfa..c179f09ad 100644 --- a/tests/general/assert-fail.c.stderr.expected +++ b/tests/general/assert-fail.c.stderr.expected @@ -2,6 +2,6 @@ Assertion failed: false (assert-fail.c: main: 5) Error: failed to run main module `assert-fail.c.---.wasm` Caused by: - 0: failed to invoke + 0: failed to invoke command default 1: error while executing at wasm backtrace: 2: wasm trap: wasm `unreachable` instruction executed diff --git a/tests/general/assert-fail.c.stderr.expected.filter b/tests/general/assert-fail.c.stderr.expected.filter index f9ffe8460..d2213dd6b 100755 --- a/tests/general/assert-fail.c.stderr.expected.filter +++ b/tests/general/assert-fail.c.stderr.expected.filter @@ -3,5 +3,4 @@ set -euo pipefail cat \ | sed -e 's/main module `assert-fail\.c\.[^`]*\.wasm`/main module `assert-fail.c.---.wasm`/' \ - | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -E '/0x[[:xdigit:]]+/d' diff --git a/tests/general/sigabrt.c.stderr.expected b/tests/general/sigabrt.c.stderr.expected index 605ac5db8..72c425d0c 100644 --- a/tests/general/sigabrt.c.stderr.expected +++ b/tests/general/sigabrt.c.stderr.expected @@ -3,4 +3,4 @@ Program received fatal signal: Aborted Error: failed to run main module `sigabrt.c.---.wasm` Caused by: - 0: failed to invoke + 0: failed to invoke command default diff --git a/tests/general/sigabrt.c.stderr.expected.filter b/tests/general/sigabrt.c.stderr.expected.filter index 0f3d78f93..425b060c2 100755 --- a/tests/general/sigabrt.c.stderr.expected.filter +++ b/tests/general/sigabrt.c.stderr.expected.filter @@ -3,6 +3,5 @@ set -euo pipefail cat \ | sed -e 's/main module `sigabrt\.c\.[^`]*\.wasm`/main module `sigabrt.c.---.wasm`/' \ - | sed -e 's/failed to invoke.*/failed to invoke/' \ | sed -e 's/source location: @[[:xdigit:]]*$/source location: @----/' \ | head -n 6