diff --git a/.circleci/config.yml b/.circleci/config.yml index f3a1c176fd3a..dff32b60d616 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -903,6 +903,16 @@ jobs: aztec_manifest_key: end-to-end <<: *defaults_e2e_test + integration-archiver-l1-to-l2: + steps: + - *checkout + - *setup_env + - run: + name: "Test" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts + aztec_manifest_key: end-to-end + <<: *defaults_e2e_test + integration-l1-publisher: steps: - *checkout @@ -1428,6 +1438,7 @@ workflows: - e2e-private-voting: *e2e_test - uniswap-trade-on-l1-from-l2: *e2e_test - integration-l1-publisher: *e2e_test + - integration-archiver-l1-to-l2: *e2e_test - e2e-persistence: *e2e_test - e2e-browser: *e2e_test - e2e-card-game: *e2e_test @@ -1493,6 +1504,7 @@ workflows: - e2e-private-voting - uniswap-trade-on-l1-from-l2 - integration-l1-publisher + - integration-archiver-l1-to-l2 - e2e-persistence - e2e-browser - e2e-card-game diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40083d1ddb22..4af800da75e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,26 @@ name: Run CI with Earthly on: push: - branches: - - master + branches: [master] pull_request: {} workflow_dispatch: {} jobs: e2e: runs-on: ubuntu-latest - strategy: { matrix: { environment: [x86], test: [e2e-escrow-contract] } } + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # TODO currently names are coupled to platform + strategy: { matrix: { environment: [arm64, amd64], test: [e2e-escrow-contract, e2e-account-contracts] } } # cancel if reran on same PR if exists, otherwise if on same commit concurrency: group: ${{ matrix.test }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }} cancel-in-progress: true steps: + - uses: earthly/actions-setup@v1 + with: + version: v0.8.5 + - name: Checkout uses: actions/checkout@v4 with: @@ -27,12 +33,23 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end - run: earthly +${{ matrix.test }} + run: | + # TODO put in script + earthly sat --org aztec launch --size 4xlarge --platform linux/${{matrix.environment}} build-${{github.actor}}-${{ matrix.environment }} || true + if [ ${{ github.ref_name }} = master ] ; then + # update the remote cache + export EARTHLY_PUSH=true + fi + # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners + earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:${{matrix.test}} --sat build-${{github.actor}}-${{ matrix.environment }} +${{ matrix.test }} bb-native-tests: runs-on: ubuntu-latest + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} # run for both x86_64 and arm64 - strategy: { matrix: { environment: [x86] } } + # TODO currently names are coupled to platform + strategy: { matrix: { environment: [amd64, arm64] } } # cancel if reran on same PR if exists, otherwise if on same commit concurrency: group: bb-native-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }} @@ -50,4 +67,12 @@ jobs: - name: Build and test working-directory: ./barretenberg/cpp - run: earthly +test + run: | + # TODO put in script + earthly sat --org aztec launch --size 4xlarge --platform linux/${{matrix.environment}} build-${{github.actor}}-${{ matrix.environment }} || true + if [ ${{ github.ref_name }} = master ] ; then + # update the remote cache + export EARTHLY_PUSH=true + fi + # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners + earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:bb-native-tests --sat build-${{github.actor}}-${{ matrix.environment }} +test diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index 49298b396e91..8dc844345ee9 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -10,7 +10,7 @@ WORKDIR /build/avm-transpiler RUN apt-get update && apt-get install -y git -COPY --keep-ts --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml . +COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml . build: RUN ./scripts/bootstrap_native.sh diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 0574c63a7108..80afe14794dc 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ WORKDIR /build SAVE IMAGE --push aztecprotocol/cache:bb-ubuntu-lunar +SAVE IMAGE --cache-hint build-wasi-sdk-image: WORKDIR / @@ -81,6 +82,7 @@ preset-wasm: DO +RUN_CMAKE --configure="--preset wasm-threads" --build="--target barretenberg.wasm" RUN ./src/wasi-sdk/bin/llvm-strip ./bin/barretenberg.wasm SAVE ARTIFACT bin + SAVE IMAGE --cache-hint preset-gcc: FROM +source @@ -132,7 +134,6 @@ test: COPY --dir ./srs_db/+build/. srs_db RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure - # Functions RUN_CMAKE: # Runs cmake build and leaves binary artifacts at 'bin' diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/mem_crs_factory.test.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/mem_crs_factory.test.cpp index bfdab78e996e..190fa75cef1d 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/mem_crs_factory.test.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/mem_crs_factory.test.cpp @@ -45,14 +45,14 @@ TEST(reference_string, mem_bn254_file_consistency) 0); } -TEST(reference_string, mem_grumpkin_file_consistency) +TEST(reference_string, DISABLED_mem_grumpkin_file_consistency) { // Load 1024 from file. - auto file_crs = FileCrsFactory("../srs_db/ignition", 1024); + auto file_crs = FileCrsFactory("../srs_db/grumpkin", 1024); // Use low level io lib to read 1024 from file. std::vector points(1024); - ::srs::IO::read_transcript_g1(points.data(), 1024, "../srs_db/ignition"); + ::srs::IO::read_transcript_g1(points.data(), 1024, "../srs_db/grumpkin"); MemGrumpkinCrsFactory mem_crs(points); auto file_prover_crs = file_crs.get_prover_crs(1024); diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile index 744a7c7b7503..9d4ef8fd373b 100644 --- a/barretenberg/ts/Earthfile +++ b/barretenberg/ts/Earthfile @@ -5,11 +5,11 @@ WORKDIR /build # minimum files to download yarn packages # keep timestamps for incremental builds -COPY --keep-ts --dir .yarn package.json yarn.lock .yarnrc.yml . +COPY --dir .yarn package.json yarn.lock .yarnrc.yml . RUN yarn --immutable # other source files -COPY --keep-ts --dir src *.json *.js *.cjs . +COPY --dir src *.json *.js *.cjs . # copy over wasm build from cpp folder COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm @@ -23,7 +23,7 @@ esm: SAVE ARTIFACT /build cjs: - COPY --keep-ts scripts/cjs_postprocess.sh scripts/ + COPY scripts/cjs_postprocess.sh scripts/ RUN yarn build:cjs SAVE ARTIFACT /build diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile index ee016750a795..6f6f42a85d26 100644 --- a/l1-contracts/Earthfile +++ b/l1-contracts/Earthfile @@ -15,7 +15,7 @@ RUN foundryup RUN npm install --global yarn solhint WORKDIR /build -COPY --keep-ts --dir lib scripts src terraform test *.json *.toml *.sh . +COPY --dir lib scripts src terraform test *.json *.toml *.sh . build: RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff diff --git a/noir/Earthfile b/noir/Earthfile index d3f86ad5fa14..54fbd8bde3b2 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -5,7 +5,7 @@ nargo: RUN apt update && apt install -y libc++1 WORKDIR /build # Relevant source (TODO finer-grained 'tooling') - COPY --keep-ts --dir \ + COPY --dir \ noir-repo/acvm-repo \ noir-repo/aztec_macros \ noir-repo/compiler \ @@ -18,10 +18,11 @@ nargo: # TODO(AD) is this OK as a content hash? ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') - COPY --keep-ts ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh + COPY ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh RUN ./scripts/bootstrap_native.sh SAVE ARTIFACT /build/noir-repo/target/release/nargo nargo SAVE ARTIFACT /build/noir-repo/target/release/acvm acvm + SAVE IMAGE aztecprotocol/nargo packages: FROM node:20 @@ -36,7 +37,7 @@ packages: WORKDIR /build # Relevant source (TODO finer-grained) - COPY --keep-ts --dir \ + COPY --dir \ noir-repo/acvm-repo \ noir-repo/aztec_macros \ noir-repo/compiler \ @@ -58,13 +59,14 @@ packages: noir-repo/.envrc \ noir-repo - COPY --keep-ts noir-repo/.github/scripts noir-repo/.github/scripts - COPY --keep-ts ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh + COPY noir-repo/.github/scripts noir-repo/.github/scripts + COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh # TODO(AD) is this OK as a content hash? ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') RUN PATH="/root/.cargo/bin:$PATH" ./scripts/bootstrap_packages.sh SAVE ARTIFACT packages + SAVE IMAGE --cache-hint run: # When running the container, mount the users home directory to same location. diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index d2dba9992591..68a98dbe823c 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -12,7 +12,7 @@ COPY ../l1-contracts/+build/out /build/l1-contracts/out WORKDIR /build/yarn-project # copy source -COPY --keep-ts --dir * *.json .yarn .yarnrc.yml . +COPY --dir * *.json .yarn .yarnrc.yml . # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index aa4758b52aed..c85026fc9262 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -25,8 +25,9 @@ E2E_TEST: FUNCTION ARG test ARG compose_file=./scripts/docker-compose.yml - LOCALLY + FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 ENV TEST=$test + COPY $compose_file $compose_file WITH DOCKER \ --load aztecprotocol/aztec:latest=+get-aztec \ --load aztecprotocol/end-to-end:latest=+get-end-to-end \ @@ -34,7 +35,6 @@ E2E_TEST: # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs RUN docker compose -f $compose_file up --exit-code-from=sandbox --force-recreate END - # we could use a parameterized target, but these just print cleaner in earthly log e2e-block-building: diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 43f30581ea34..ce7fd63d57fc 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -5,12 +5,10 @@ services: entrypoint: > sh -c ' if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - { anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" ; echo $$? > .status ; } | grep -v eth_getLogs + anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" else - { anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 ; echo $$? > .status ; } | grep -v eth_getLogs - fi - # final status - grep -q '^0$$' .status' + anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 + fi' ports: - '8545:8545'