From e8cf57937e7b80e0b29a5406d7bd387e4e427243 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 14:44:39 +0000 Subject: [PATCH 01/16] e2e on spots [ci rebuild] --- .circleci/config.yml | 62 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79993c70dc7e..9f7d13e36263 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -612,7 +612,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_2_pxes.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_2_pxes.test.ts environment: { DEBUG: "aztec:*" } e2e-multiple-accounts-1-enc-key: @@ -624,7 +624,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts environment: { DEBUG: "aztec:*" } e2e-deploy-contract: @@ -636,7 +636,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_deploy_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_deploy_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-lending-contract: @@ -648,7 +648,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_lending_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_lending_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-token-contract: @@ -660,7 +660,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_token_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_token_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-sandbox-example: @@ -672,7 +672,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_sandbox_example.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_sandbox_example.test.ts environment: { DEBUG: "aztec:*" } e2e-block-building: @@ -684,7 +684,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_block_building.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_block_building.test.ts environment: { DEBUG: "aztec:*" } e2e-nested-contract: @@ -696,7 +696,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_nested_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_nested_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-non-contract-account: @@ -708,7 +708,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_non_contract_account.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_non_contract_account.test.ts environment: { DEBUG: "aztec:*" } e2e-cross-chain-messaging: @@ -720,7 +720,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-cross-chain-messaging: @@ -732,7 +732,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-to-private-messaging: @@ -744,7 +744,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-account-contracts: @@ -756,7 +756,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_account_contracts.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_account_contracts.test.ts environment: { DEBUG: "aztec:*" } e2e-escrow-contract: @@ -768,7 +768,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_escrow_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_escrow_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-pending-commitments-contract: @@ -780,7 +780,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-ordering: @@ -792,7 +792,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_ordering.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_ordering.test.ts environment: { DEBUG: "aztec:*" } uniswap-trade-on-l1-from-l2: @@ -804,7 +804,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts environment: { DEBUG: "aztec:*" } integration-archiver-l1-to-l2: @@ -816,7 +816,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts environment: { DEBUG: "aztec:*" } integration-l1-publisher: @@ -828,7 +828,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_l1_publisher.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local integration_l1_publisher.test.ts environment: { DEBUG: "aztec:*" } e2e-cli: @@ -840,7 +840,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cli.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_cli.test.ts environment: { DEBUG: "aztec:*" } e2e-p2p: @@ -852,7 +852,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_p2p_network.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_p2p_network.test.ts ./scripts/docker-compose-no-sandbox.yml environment: { DEBUG: "aztec:*" } e2e-browser-sandbox: @@ -864,7 +864,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-browser.yml + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-browser.yml environment: { DEBUG: "aztec:*" } e2e-card-game: @@ -876,7 +876,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_card_game.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_card_game.test.ts environment: { DEBUG: "aztec:*" } pxe-sandbox: @@ -888,7 +888,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local pxe_sandbox.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local pxe_sandbox.test.ts environment: { DEBUG: "aztec:*" } cli-docs-sandbox: @@ -900,7 +900,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local cli_docs_sandbox.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local cli_docs_sandbox.test.ts environment: { DEBUG: "aztec:*" } guides-writing-an-account-contract: @@ -912,7 +912,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/writing_an_account_contract.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/writing_an_account_contract.test.ts environment: { DEBUG: "aztec:*" } guides-dapp-testing: @@ -924,7 +924,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/dapp_testing.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/dapp_testing.test.ts environment: { DEBUG: "aztec:*" } guides-sample-dapp: @@ -936,7 +936,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local sample-dapp + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local sample-dapp environment: { DEBUG: "aztec:*" } guides-up-quick-start: @@ -948,7 +948,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/up_quick_start.test.ts + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/up_quick_start.test.ts environment: { DEBUG: "aztec:*" } bench-publish-rollup: @@ -960,7 +960,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts ./scripts/docker-compose-no-sandbox.yml environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" @@ -973,7 +973,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_process_history.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local benchmarks/bench_process_history.test.ts ./scripts/docker-compose-no-sandbox.yml environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" From 5130412f4efb337c7425a3928c40c390ce7286b7 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 16:33:51 +0000 Subject: [PATCH 02/16] Run e2e test via new build system compose script. --- .circleci/config.yml | 64 +++++++++++----------- build-system/scripts/build | 2 +- build-system/scripts/cond_run_compose | 35 ++++++++++++ build-system/scripts/cond_spot_run_compose | 10 ++++ build-system/scripts/query_manifest | 2 - build-system/scripts/setup_env | 3 +- build-system/scripts/upload_logs_to_s3 | 31 +++++++++++ 7 files changed, 111 insertions(+), 36 deletions(-) create mode 100755 build-system/scripts/cond_run_compose create mode 100755 build-system/scripts/cond_spot_run_compose create mode 100755 build-system/scripts/upload_logs_to_s3 diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f7d13e36263..9c2ed3d6d8bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,7 +66,7 @@ checkout: &checkout setup_env: &setup_env run: name: "Setup environment" - command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" + command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" # This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them save_logs: &save_logs @@ -612,7 +612,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_2_pxes.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_2_pxes.test.ts environment: { DEBUG: "aztec:*" } e2e-multiple-accounts-1-enc-key: @@ -624,7 +624,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_multiple_accounts_1_enc_key.test.ts environment: { DEBUG: "aztec:*" } e2e-deploy-contract: @@ -636,7 +636,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_deploy_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_deploy_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-lending-contract: @@ -648,7 +648,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_lending_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_lending_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-token-contract: @@ -660,7 +660,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_token_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_token_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-sandbox-example: @@ -672,7 +672,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_sandbox_example.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_sandbox_example.test.ts environment: { DEBUG: "aztec:*" } e2e-block-building: @@ -684,7 +684,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_block_building.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_block_building.test.ts environment: { DEBUG: "aztec:*" } e2e-nested-contract: @@ -696,7 +696,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_nested_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_nested_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-non-contract-account: @@ -708,7 +708,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_non_contract_account.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_non_contract_account.test.ts environment: { DEBUG: "aztec:*" } e2e-cross-chain-messaging: @@ -720,7 +720,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-cross-chain-messaging: @@ -732,7 +732,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_public_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-to-private-messaging: @@ -744,7 +744,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_public_to_private_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-account-contracts: @@ -756,7 +756,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_account_contracts.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_account_contracts.test.ts environment: { DEBUG: "aztec:*" } e2e-escrow-contract: @@ -768,7 +768,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_escrow_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_escrow_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-pending-commitments-contract: @@ -780,7 +780,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_pending_commitments_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-ordering: @@ -792,7 +792,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_ordering.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_ordering.test.ts environment: { DEBUG: "aztec:*" } uniswap-trade-on-l1-from-l2: @@ -804,7 +804,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=uniswap_trade_on_l1_from_l2.test.ts environment: { DEBUG: "aztec:*" } integration-archiver-l1-to-l2: @@ -816,7 +816,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=integration_archiver_l1_to_l2.test.ts environment: { DEBUG: "aztec:*" } integration-l1-publisher: @@ -828,7 +828,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local integration_l1_publisher.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=integration_l1_publisher.test.ts environment: { DEBUG: "aztec:*" } e2e-cli: @@ -840,7 +840,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_cli.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_cli.test.ts environment: { DEBUG: "aztec:*" } e2e-p2p: @@ -852,7 +852,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_p2p_network.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=e2e_p2p_network.test.ts environment: { DEBUG: "aztec:*" } e2e-browser-sandbox: @@ -864,7 +864,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-browser.yml + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml -e TEST=e2e_aztec_js_browser.test.ts environment: { DEBUG: "aztec:*" } e2e-card-game: @@ -876,7 +876,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local e2e_card_game.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_card_game.test.ts environment: { DEBUG: "aztec:*" } pxe-sandbox: @@ -888,7 +888,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local pxe_sandbox.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=pxe_sandbox.test.ts environment: { DEBUG: "aztec:*" } cli-docs-sandbox: @@ -900,7 +900,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local cli_docs_sandbox.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=cli_docs_sandbox.test.ts environment: { DEBUG: "aztec:*" } guides-writing-an-account-contract: @@ -912,7 +912,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/writing_an_account_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/writing_an_account_contract.test.ts environment: { DEBUG: "aztec:*" } guides-dapp-testing: @@ -924,7 +924,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/dapp_testing.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/dapp_testing.test.ts environment: { DEBUG: "aztec:*" } guides-sample-dapp: @@ -936,7 +936,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local sample-dapp + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=sample-dapp environment: { DEBUG: "aztec:*" } guides-up-quick-start: @@ -948,7 +948,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local guides/up_quick_start.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/up_quick_start.test.ts environment: { DEBUG: "aztec:*" } bench-publish-rollup: @@ -960,7 +960,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=benchmarks/bench_publish_rollup.test.ts environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" @@ -973,7 +973,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_spot_run_script end-to-end 4 ./scripts/run_tests_local benchmarks/bench_process_history.test.ts ./scripts/docker-compose-no-sandbox.yml + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=benchmarks/bench_process_history.test.ts environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" diff --git a/build-system/scripts/build b/build-system/scripts/build index ff7c81d82b02..904ed2b31079 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -105,7 +105,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. TAG=$ECR_DEPLOY_URL/$PARENT_REPO - retry docker tag $PARENT_IMAGE_URI $TAG + docker tag $PARENT_IMAGE_URI $TAG done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose new file mode 100755 index 000000000000..ca26c7fdeba8 --- /dev/null +++ b/build-system/scripts/cond_run_compose @@ -0,0 +1,35 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +COMPOSE_FILE=$2 +shift 2 + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +BASE_TAG=cache-$CONTENT_HASH +SUCCESS_TAG=$BASE_TAG-$JOB_NAME + +echo "Content hash: $CONTENT_HASH" + +if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then + # Login to pull our ecr images with docker. + ecr_login + + # For each dependency and self, pull in the latest image and give it correct tag. + for REPO in $(query_manifest dependencies $REPOSITORY) $REPOSITORY; do + echo "Pulling $REPO..." + REPO_IMAGE_URI=$(calculate_image_uri $REPO) + retry docker pull $REPO_IMAGE_URI + docker tag $REPO_IMAGE_URI aztecprotocol/$REPO + done + + cd $(query_manifest projectDir) + + docker-compose -f $COMPOSE_FILE rm -f + docker-compose -f $COMPOSE_FILE up --exit-code-from $REPOSITORY $@ + + upload_logs_to_s3 log + + retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG +fi diff --git a/build-system/scripts/cond_spot_run_compose b/build-system/scripts/cond_spot_run_compose new file mode 100755 index 000000000000..b10cc9decf59 --- /dev/null +++ b/build-system/scripts/cond_spot_run_compose @@ -0,0 +1,10 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +CPUS=$2 +shift 2 + +export TAG_POSTFIX=$JOB_NAME +cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@ diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index edf5f62b6fd9..dd2f201631f5 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -94,8 +94,6 @@ case "$CMD" in ALL_DEPS+=("$1") } add_deps $REPO - # Remove ourself as a dependency. - # unset ALL_DEPS[-1] for DEP in "${ALL_DEPS[@]}"; do echo $DEP done diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 8e436a9ca8bb..60880473baca 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -13,7 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-} JOB_NAME=$3 GIT_REPOSITORY_URL=${4:-} BRANCH=${5:-} -COMMIT_MESSAGE=${6:-} +PULL_REQUEST=${6:-} BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) @@ -87,6 +87,7 @@ echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV +echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV # Only run the following if we're the result of a commit (i.e. not being run manually). if [ -n "$COMMIT_HASH" ]; then diff --git a/build-system/scripts/upload_logs_to_s3 b/build-system/scripts/upload_logs_to_s3 new file mode 100755 index 000000000000..ede934edae59 --- /dev/null +++ b/build-system/scripts/upload_logs_to_s3 @@ -0,0 +1,31 @@ +#!/bin/bash + +# Uploads to S3 the contents of the log file mounted on the end-to-end container, +# which contains log entries with an associated event and metrics for it. +# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl +# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +LOG_FOLDER=$1 +BUCKET_NAME="aztec-ci-artifacts" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then + echo "No logs in folder $LOG_FOLDER to upload" + exit 0 +fi + +# Duplicated in scripts/ci/assemble_e2e_benchmark.sh +if [ "${BRANCH:-}" = "master" ]; then + TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" +fi + +if [ -n "${TARGET_FOLDER:-}" ]; then + aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive +else + echo Skipping upload since no target folder was defined +fi \ No newline at end of file From 472bebde3c7dd338a35731c560f8ea0d3e71e704 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 16:41:25 +0000 Subject: [PATCH 03/16] [ci rebuild end-to-end] From c91cf282f24cc24a38e3b5fa542c052bc1752cf0 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 17:40:56 +0000 Subject: [PATCH 04/16] wip --- build-system/scripts/cond_run_compose | 4 ++-- build-system/scripts/query_manifest | 6 ++++++ build-system/scripts/remote_runner | 2 +- build_manifest.yml | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index ca26c7fdeba8..474cee6892eb 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -17,14 +17,14 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then ecr_login # For each dependency and self, pull in the latest image and give it correct tag. - for REPO in $(query_manifest dependencies $REPOSITORY) $REPOSITORY; do + for REPO in $(query_manifest runDependencies $REPOSITORY) $REPOSITORY; do echo "Pulling $REPO..." REPO_IMAGE_URI=$(calculate_image_uri $REPO) retry docker pull $REPO_IMAGE_URI docker tag $REPO_IMAGE_URI aztecprotocol/$REPO done - cd $(query_manifest projectDir) + cd $(query_manifest projectDir $REPOSITORY) docker-compose -f $COMPOSE_FILE rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from $REPOSITORY $@ diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index dd2f201631f5..0358fbf89cb7 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -75,6 +75,12 @@ case "$CMD" in dependencies) yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST ;; + runDependencies) + echo $REPO + RUN_NAME=$3 + yq -r ".\"$REPO\".runDependencies // [] | .[]" $MANIFEST + yq -r ".\"$REPO\".run.\"$RUN_NAME\".dependencies // [] | .[]" $MANIFEST + ;; allDependencies) # Get dependencies for a given repo. Inclusive of repo itself. # If a string, attempt to execute / if exists to get dependencies, else error. diff --git a/build-system/scripts/remote_runner b/build-system/scripts/remote_runner index 243d98e47bf5..ce1a567a1989 100755 --- a/build-system/scripts/remote_runner +++ b/build-system/scripts/remote_runner @@ -22,6 +22,6 @@ echo "Git checkout completed." BASH_ENV=/tmp/bash_env echo "Calling setup env..." -source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" +source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST" echo "Calling $@..." $@ diff --git a/build_manifest.yml b/build_manifest.yml index dbacc1f9262a..c9aa68cf6a5f 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -166,6 +166,8 @@ end-to-end: projectDir: yarn-project/end-to-end dependencies: - yarn-project + runDependencies: + - aztec-sandbox aztec-node: buildDir: yarn-project From f459652eada3de5c3ff1ba6d581ba1e1a5089c1c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 17:56:17 +0000 Subject: [PATCH 05/16] [ci rebuild end-to-end] From ca4d4b9335b2910f9caf2cd02b752ffd39314545 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 18:25:35 +0000 Subject: [PATCH 06/16] Fix. --- build-system/scripts/remote_run_script | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 5ad1a850e141..8aa4e0ea64f4 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -24,6 +24,8 @@ ssh -A -F $SSH_CONFIG_PATH $IP " export COMMIT_TAG=$COMMIT_TAG export JOB_NAME=$JOB_NAME export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL + export BRANCH=${BRANCH:-} + export PULL_REQUEST=${PULL_REQUEST:-} export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD export ECR_DEPLOY_URL=$ECR_DEPLOY_URL export ECR_URL=$ECR_URL From de969ed10915beded7da01f1534263539a2710da Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 19:01:37 +0000 Subject: [PATCH 07/16] [ci rebuild end-to-end] From cd7df82a380b64d4286609414e13340c6303bb33 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 21:01:27 +0000 Subject: [PATCH 08/16] [ci rebuild end-to-end] From 91f1de34d5ef6880d78f05a61fc2a21ef07a0234 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 21:06:29 +0000 Subject: [PATCH 09/16] [ci rebuild end-to-end] --- build-system/scripts/cond_run_compose | 2 +- yarn-project/end-to-end/scripts/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 474cee6892eb..5f7b2d84f836 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -17,7 +17,7 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then ecr_login # For each dependency and self, pull in the latest image and give it correct tag. - for REPO in $(query_manifest runDependencies $REPOSITORY) $REPOSITORY; do + for REPO in $(query_manifest runDependencies $REPOSITORY $JOB_NAME) $REPOSITORY; do echo "Pulling $REPO..." REPO_IMAGE_URI=$(calculate_image_uri $REPO) retry docker pull $REPO_IMAGE_URI diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index ed5880dad89d..2b6ad88cf97b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' LOG_LEVL: 'debug' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 From 1e40800846b229d23368f045bfcba9f03d579354 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 21:37:58 +0000 Subject: [PATCH 10/16] [ci rebuild end-to-end] --- build-system/scripts/cond_run_compose | 3 +-- yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml | 2 ++ yarn-project/end-to-end/scripts/docker-compose.yml | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 5f7b2d84f836..98bbd99d81ea 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -26,8 +26,7 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then cd $(query_manifest projectDir $REPOSITORY) - docker-compose -f $COMPOSE_FILE rm -f - docker-compose -f $COMPOSE_FILE up --exit-code-from $REPOSITORY $@ + docker-compose -f $COMPOSE_FILE run $@ $REPOSITORY upload_logs_to_s3 log diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index bd2020709e5f..133b8c8d055b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -28,3 +28,5 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - fork diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 2b6ad88cf97b..85c90b07cd66 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -39,3 +39,6 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork From 9a3ed1cab8e598e561c9fca2ff3657ab87e38a40 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 22:36:46 +0000 Subject: [PATCH 11/16] [ci rebuild end-to-end] --- .circleci/config.yml | 62 +++++++++---------- build-system/scripts/cond_run_compose | 2 +- .../scripts/docker-compose-no-sandbox.yml | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c2ed3d6d8bc..ec8a71122724 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -612,7 +612,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_2_pxes.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts environment: { DEBUG: "aztec:*" } e2e-multiple-accounts-1-enc-key: @@ -624,7 +624,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_multiple_accounts_1_enc_key.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts environment: { DEBUG: "aztec:*" } e2e-deploy-contract: @@ -636,7 +636,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_deploy_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-lending-contract: @@ -648,7 +648,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_lending_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-token-contract: @@ -660,7 +660,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_token_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-sandbox-example: @@ -672,7 +672,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_sandbox_example.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts environment: { DEBUG: "aztec:*" } e2e-block-building: @@ -684,7 +684,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_block_building.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts environment: { DEBUG: "aztec:*" } e2e-nested-contract: @@ -696,7 +696,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_nested_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-non-contract-account: @@ -708,7 +708,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_non_contract_account.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts environment: { DEBUG: "aztec:*" } e2e-cross-chain-messaging: @@ -720,7 +720,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_cross_chain_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-cross-chain-messaging: @@ -732,7 +732,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_public_cross_chain_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-public-to-private-messaging: @@ -744,7 +744,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_public_to_private_messaging.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts environment: { DEBUG: "aztec:*" } e2e-account-contracts: @@ -756,7 +756,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_account_contracts.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts environment: { DEBUG: "aztec:*" } e2e-escrow-contract: @@ -768,7 +768,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_escrow_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-pending-commitments-contract: @@ -780,7 +780,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_pending_commitments_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts environment: { DEBUG: "aztec:*" } e2e-ordering: @@ -792,7 +792,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_ordering.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts environment: { DEBUG: "aztec:*" } uniswap-trade-on-l1-from-l2: @@ -804,7 +804,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=uniswap_trade_on_l1_from_l2.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts environment: { DEBUG: "aztec:*" } integration-archiver-l1-to-l2: @@ -816,7 +816,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=integration_archiver_l1_to_l2.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts environment: { DEBUG: "aztec:*" } integration-l1-publisher: @@ -828,7 +828,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=integration_l1_publisher.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts environment: { DEBUG: "aztec:*" } e2e-cli: @@ -840,7 +840,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_cli.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts environment: { DEBUG: "aztec:*" } e2e-p2p: @@ -852,7 +852,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=e2e_p2p_network.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts environment: { DEBUG: "aztec:*" } e2e-browser-sandbox: @@ -864,7 +864,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml -e TEST=e2e_aztec_js_browser.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts environment: { DEBUG: "aztec:*" } e2e-card-game: @@ -876,7 +876,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=e2e_card_game.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts environment: { DEBUG: "aztec:*" } pxe-sandbox: @@ -888,7 +888,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=pxe_sandbox.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts environment: { DEBUG: "aztec:*" } cli-docs-sandbox: @@ -900,7 +900,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=cli_docs_sandbox.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts environment: { DEBUG: "aztec:*" } guides-writing-an-account-contract: @@ -912,7 +912,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/writing_an_account_contract.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts environment: { DEBUG: "aztec:*" } guides-dapp-testing: @@ -924,7 +924,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/dapp_testing.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts environment: { DEBUG: "aztec:*" } guides-sample-dapp: @@ -936,7 +936,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=sample-dapp + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp environment: { DEBUG: "aztec:*" } guides-up-quick-start: @@ -948,7 +948,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml -e TEST=guides/up_quick_start.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts environment: { DEBUG: "aztec:*" } bench-publish-rollup: @@ -960,7 +960,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=benchmarks/bench_publish_rollup.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" @@ -973,7 +973,7 @@ jobs: - *setup_env - run: name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml -e TEST=benchmarks/bench_process_history.test.ts + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts environment: DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 98bbd99d81ea..b3a11edac135 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -26,7 +26,7 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then cd $(query_manifest projectDir $REPOSITORY) - docker-compose -f $COMPOSE_FILE run $@ $REPOSITORY + $@ docker-compose -f $COMPOSE_FILE run $REPOSITORY upload_logs_to_s3 log diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index 133b8c8d055b..76d045ebc4bd 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -15,7 +15,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 From 7a7db3db463efab1f04cbb2b011e12697d425574 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 22:42:32 +0000 Subject: [PATCH 12/16] [ci rebuild end-to-end] --- build-system/scripts/cond_run_compose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index b3a11edac135..58eef99c71a7 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -26,7 +26,8 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then cd $(query_manifest projectDir $REPOSITORY) - $@ docker-compose -f $COMPOSE_FILE run $REPOSITORY + export $@ + docker-compose -f $COMPOSE_FILE run $REPOSITORY upload_logs_to_s3 log From ea1d3f51d7d5662bed6b68ee34e18bd10e2f4d31 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 23:10:47 +0000 Subject: [PATCH 13/16] [ci rebuild end-to-end] --- .circleci/config.yml | 285 ++++++++---------- cspell.json | 9 +- .../scripts/docker-compose-browser.yml | 5 +- 3 files changed, 136 insertions(+), 163 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec8a71122724..7acc0050f007 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ # The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download # with just enough tools to execute the build system, and launch EC2 instances etc. # -# There are some `join` steps that are just noops. They are just used to produce cleaner graph rendering in CCI. +# There are some `join` steps that are just no-ops. They are just used to produce cleaner graph rendering in CCI. version: 2.1 @@ -61,7 +61,7 @@ checkout: &checkout retry git checkout FETCH_HEAD # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. -# It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of +# It takes the required CCI environment variables as inputs, and gives them normalized names for the rest of # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds. setup_env: &setup_env run: @@ -604,378 +604,345 @@ jobs: command: build end-to-end e2e-2-pxes: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts DEBUG=aztec:* e2e-multiple-accounts-1-enc-key: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts DEBUG=aztec:* e2e-deploy-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts DEBUG=aztec:* e2e-lending-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts DEBUG=aztec:* e2e-token-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts DEBUG=aztec:* e2e-sandbox-example: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts DEBUG=aztec:* e2e-block-building: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts DEBUG=aztec:* e2e-nested-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts DEBUG=aztec:* e2e-non-contract-account: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts DEBUG=aztec:* e2e-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts DEBUG=aztec:* e2e-public-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts DEBUG=aztec:* e2e-public-to-private-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts DEBUG=aztec:* e2e-account-contracts: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts DEBUG=aztec:* e2e-escrow-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts DEBUG=aztec:* e2e-pending-commitments-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts DEBUG=aztec:* e2e-ordering: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts DEBUG=aztec:* uniswap-trade-on-l1-from-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts DEBUG=aztec:* integration-archiver-l1-to-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts DEBUG=aztec:* integration-l1-publisher: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts DEBUG=aztec:* e2e-cli: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts DEBUG=aztec:* e2e-p2p: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts DEBUG=aztec:* e2e-browser-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts DEBUG=aztec:* e2e-card-game: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts DEBUG=aztec:* pxe-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts DEBUG=aztec:* cli-docs-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts DEBUG=aztec:* guides-writing-an-account-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts DEBUG=aztec:* guides-dapp-testing: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts DEBUG=aztec:* guides-sample-dapp: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp" DEBUG=aztec:* guides-up-quick-start: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts"" DEBUG=aztec:* bench-publish-rollup: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees bench-process-history: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees build-docs: machine: diff --git a/cspell.json b/cspell.json index b4ae2cbad6dd..00506c7af678 100644 --- a/cspell.json +++ b/cspell.json @@ -27,6 +27,8 @@ "camelcase", "cbind", "cbinds", + "cimg", + "cpus", "chainsafe", "cheatcode", "cheatcodes", @@ -37,6 +39,7 @@ "comlink", "composability", "concat", + "cond", "customizability", "danlee", "Daos", @@ -73,6 +76,7 @@ "Kademlia", "keccak", "keypairs", + "keyscan", "leveldb", "leveldown", "leveljs", @@ -132,6 +136,7 @@ "sload", "snakecase", "solhint", + "stdlib", "struct", "structs", "subrepo", @@ -178,7 +183,5 @@ "*.snap", "package.json" ], - "flagWords": [ - "anonymous" - ] + "flagWords": ["anonymous"] } diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index 5d1e8dbc13b5..ddb5838000b6 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -38,3 +38,6 @@ services: entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork From 175ba605b1ffdc00363acdf7f2c016b9c194c1ea Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 23:22:40 +0000 Subject: [PATCH 14/16] [ci rebuild end-to-end] --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7acc0050f007..538132698682 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -909,7 +909,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp" DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp DEBUG=aztec:* guides-up-quick-start: docker: @@ -920,7 +920,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts"" DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts DEBUG=aztec:* bench-publish-rollup: docker: From 809be00e410af5661d51b5de2cfad4665e71fb57 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 23:46:30 +0000 Subject: [PATCH 15/16] [ci rebuild end-to-end] --- .circleci/config.yml | 58 +++++++++---------- .../end-to-end/scripts/cond_run_script.delme | 34 ----------- yarn-project/end-to-end/scripts/run_tests | 20 ------- .../end-to-end/scripts/run_tests_local | 24 -------- .../end-to-end/scripts/upload_logs_to_s3.sh | 31 ---------- 5 files changed, 29 insertions(+), 138 deletions(-) delete mode 100755 yarn-project/end-to-end/scripts/cond_run_script.delme delete mode 100755 yarn-project/end-to-end/scripts/run_tests delete mode 100755 yarn-project/end-to-end/scripts/run_tests_local delete mode 100755 yarn-project/end-to-end/scripts/upload_logs_to_s3.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 538132698682..edecd9b086df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -612,7 +612,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts e2e-multiple-accounts-1-enc-key: docker: @@ -623,7 +623,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts e2e-deploy-contract: docker: @@ -634,7 +634,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts e2e-lending-contract: docker: @@ -645,7 +645,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts e2e-token-contract: docker: @@ -656,7 +656,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts e2e-sandbox-example: docker: @@ -667,7 +667,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts e2e-block-building: docker: @@ -678,7 +678,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts e2e-nested-contract: docker: @@ -689,7 +689,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts e2e-non-contract-account: docker: @@ -700,7 +700,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts e2e-cross-chain-messaging: docker: @@ -711,7 +711,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts e2e-public-cross-chain-messaging: docker: @@ -722,7 +722,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts e2e-public-to-private-messaging: docker: @@ -733,7 +733,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts e2e-account-contracts: docker: @@ -744,7 +744,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts e2e-escrow-contract: docker: @@ -755,7 +755,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts e2e-pending-commitments-contract: docker: @@ -766,7 +766,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts e2e-ordering: docker: @@ -777,7 +777,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts uniswap-trade-on-l1-from-l2: docker: @@ -788,7 +788,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts integration-archiver-l1-to-l2: docker: @@ -799,7 +799,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts integration-l1-publisher: docker: @@ -810,7 +810,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts e2e-cli: docker: @@ -821,7 +821,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts e2e-p2p: docker: @@ -832,7 +832,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts e2e-browser-sandbox: docker: @@ -843,7 +843,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts e2e-card-game: docker: @@ -854,7 +854,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts pxe-sandbox: docker: @@ -865,7 +865,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts cli-docs-sandbox: docker: @@ -876,7 +876,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts guides-writing-an-account-contract: docker: @@ -887,7 +887,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts guides-dapp-testing: docker: @@ -898,7 +898,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts guides-sample-dapp: docker: @@ -909,7 +909,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp guides-up-quick-start: docker: @@ -920,7 +920,7 @@ jobs: - *setup_env - run: name: "Test" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts DEBUG=aztec:* + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts bench-publish-rollup: docker: diff --git a/yarn-project/end-to-end/scripts/cond_run_script.delme b/yarn-project/end-to-end/scripts/cond_run_script.delme deleted file mode 100755 index df4292891375..000000000000 --- a/yarn-project/end-to-end/scripts/cond_run_script.delme +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Conditionally runs a script if any dependent code has changed between -# the last successful run and the present commit. -# -# It's expected to be run from the project directory, and that there be a directory called `scripts` -# containing the given named script to execute. -# -# This script is only useful if there is nothing to do in the event there is no rebuild. This is fine -# for running a suite of tests for example, but is not useful for performing a build, as even if a -# build has nothing to do, the previous images are retagged with the new commit hash for upstream jobs. -# -# Arguments are: -# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a -# successful run. The script will only run if there were relevant code changes since the last successful commit. -# 3... ARGS: Script to run and args. -set -eu -set -x - -REPOSITORY=$1 -shift -SUCCESS_TAG=$1 -shift -SCRIPT_TO_RUN=$1 -shift - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" -echo "Script to run is $SCRIPT_TO_RUN $@" - -if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - "$SCRIPT_TO_RUN" "$@" - retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG -fi diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests deleted file mode 100755 index 97b1effcc9e4..000000000000 --- a/yarn-project/end-to-end/scripts/run_tests +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-docker-compose.yml} - -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in aztec.js end-to-end aztec-sandbox; do - retry docker pull $($calculate_image_uri $REPO) - retry docker tag $($calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local deleted file mode 100755 index 3df64ff40616..000000000000 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_run_script). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} - -# Login to ECR. -ecr_login - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in end-to-end aztec-sandbox; do - retry docker pull $(calculate_image_uri $REPO) - retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose -f $COMPOSE_FILE rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end - -# Upload the logs with metrics to S3 -./scripts/upload_logs_to_s3.sh \ No newline at end of file diff --git a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh b/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh deleted file mode 100755 index eb81df8bfac2..000000000000 --- a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Uploads to S3 the contents of the log file mounted on the end-to-end container, -# which contains log entries with an associated event and metrics for it. -# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl -# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -BUCKET_NAME="aztec-ci-artifacts" -LOG_FOLDER="${LOG_FOLDER:-log}" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" - -if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then - echo "No logs in folder $LOG_FOLDER to upload" - exit 0 -fi - -# Duplicated in scripts/ci/assemble_e2e_benchmark.sh -if [ "${CIRCLE_BRANCH:-}" = "master" ]; then - TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" -elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" -fi - -if [ -n "${TARGET_FOLDER:-}" ]; then - aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive -else - echo Skipping upload since no target folder was defined -fi \ No newline at end of file From 87542dde523f9092bd461593c353d7e4e201620a Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Thu, 26 Oct 2023 23:47:05 +0000 Subject: [PATCH 16/16] [ci rebuild end-to-end]