From c77dff43ee7e8f87ee10d84d105d7fbb5dc73416 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:03:35 +0000 Subject: [PATCH 1/7] test: run check rebuild on boxes --- yarn-project/boxes/blank/run_tests | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/yarn-project/boxes/blank/run_tests b/yarn-project/boxes/blank/run_tests index 5037c7617cb8..4c2ae3cb5fd8 100755 --- a/yarn-project/boxes/blank/run_tests +++ b/yarn-project/boxes/blank/run_tests @@ -12,6 +12,18 @@ export COMPOSE_FILE=${1:-docker-compose.yml} ecr_login REPO=aztec-sandbox + +ensure_repo $REPO $ECR_REGION refresh_lifecycle +CONTENT_HASH=$(calculate_content_hash $REPO) +BASE_TAG=cache-$CONTENT_HASH +SUCCESS_TAG=$BASE_TAG + +echo "Content hash: $CONTENT_HASH" +if check_rebuild $SUCCESS_TAG $REPO; then + echo "No rebuild required." + exit 0 +fi + echo "pulling docker image for $REPO $(calculate_image_uri $REPO)-x86_64" retry docker pull $(calculate_image_uri $REPO)-x86_64 retry docker tag $(calculate_image_uri $REPO)-x86_64 aztecprotocol/$REPO:latest From 39ddefa86a3f53906466b1d9cad941230f046551 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 20:33:50 +0000 Subject: [PATCH 2/7] exp: unify script --- .circleci/config.yml | 6 +++--- yarn-project/boxes/blank-react/run_tests | 24 ------------------------ yarn-project/boxes/{blank => }/run_tests | 6 ++++-- yarn-project/boxes/token/run_tests | 24 ------------------------ 4 files changed, 7 insertions(+), 53 deletions(-) delete mode 100755 yarn-project/boxes/blank-react/run_tests rename yarn-project/boxes/{blank => }/run_tests (93%) delete mode 100755 yarn-project/boxes/token/run_tests diff --git a/.circleci/config.yml b/.circleci/config.yml index 9bf6b163fd75..1aa29858502a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -546,7 +546,7 @@ jobs: - *setup_env - run: name: "Test via adhoc script" - command: ./yarn-project/boxes/blank-react/run_tests + command: ./yarn-project/boxes/run_tests blank-react boxes-blank: machine: @@ -557,7 +557,7 @@ jobs: - *setup_env - run: name: "Test via adhoc script" - command: ./yarn-project/boxes/blank/run_tests + command: ./yarn-project/boxes/run_tests blank boxes-token: machine: @@ -568,7 +568,7 @@ jobs: - *setup_env - run: name: "Test via adhoc script" - command: ./yarn-project/boxes/token/run_tests + command: ./yarn-project/boxes/run_tests token canary: machine: diff --git a/yarn-project/boxes/blank-react/run_tests b/yarn-project/boxes/blank-react/run_tests deleted file mode 100755 index 86800b07a905..000000000000 --- a/yarn-project/boxes/blank-react/run_tests +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see .circleci/config.yml). -# It pulls images and runs docker-compose, which has the test as the entrypoint. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -# Run in script path. -cd `dirname $0` - -export COMPOSE_FILE=${1:-docker-compose.yml} - -ecr_login -REPO=aztec-sandbox -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)-x86_64" -retry docker pull $(calculate_image_uri $REPO)-x86_64 -retry docker tag $(calculate_image_uri $REPO)-x86_64 aztecprotocol/$REPO:latest - -REPO=yarn-project -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)" -retry docker pull $(calculate_image_uri $REPO) -retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest - -docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-blank-react \ No newline at end of file diff --git a/yarn-project/boxes/blank/run_tests b/yarn-project/boxes/run_tests similarity index 93% rename from yarn-project/boxes/blank/run_tests rename to yarn-project/boxes/run_tests index 4c2ae3cb5fd8..423ba9385be4 100755 --- a/yarn-project/boxes/blank/run_tests +++ b/yarn-project/boxes/run_tests @@ -4,8 +4,10 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu -# Run in script path. -cd `dirname $0` +CURRENT_DIR=`dirname $0` +BOX_NAME=${1:-boxes-blank} + +cd $CURRENT_DIR/$BOX_NAME export COMPOSE_FILE=${1:-docker-compose.yml} diff --git a/yarn-project/boxes/token/run_tests b/yarn-project/boxes/token/run_tests deleted file mode 100755 index f4b797520e8a..000000000000 --- a/yarn-project/boxes/token/run_tests +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e type test in CI (see .circleci/config.yml). -# It pulls images and runs docker-compose, which has the test as the entrypoint. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -# Run in script path. -cd `dirname $0` - -export COMPOSE_FILE=${1:-docker-compose.yml} - -ecr_login -REPO=aztec-sandbox -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)-x86_64" -retry docker pull $(calculate_image_uri $REPO)-x86_64 -retry docker tag $(calculate_image_uri $REPO)-x86_64 aztecprotocol/$REPO:latest - -REPO=yarn-project -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)" -retry docker pull $(calculate_image_uri $REPO) -retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest - -docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-token \ No newline at end of file From cd0a35d452805b48d356bb5aea854fad2bde42b3 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 20:41:59 +0000 Subject: [PATCH 3/7] pls rebuild --- yarn-project/yarn-project-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 4d92493bd06f..f5b60bbda1a2 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -46,6 +46,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts FROM node:18-alpine RUN apk update && apk add --no-cache bash jq curl + WORKDIR /usr/src/yarn-project # The dockerignore file ensures the context only contains package.json and tsconfig.json files. From 362d757e4e8723d7321e4ac1489f32fa1ce615e2 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 20:52:26 +0000 Subject: [PATCH 4/7] oof --- yarn-project/boxes/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/boxes/run_tests b/yarn-project/boxes/run_tests index 423ba9385be4..6fc24364a0fd 100755 --- a/yarn-project/boxes/run_tests +++ b/yarn-project/boxes/run_tests @@ -36,4 +36,4 @@ retry docker pull $(calculate_image_uri $REPO) retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-blank +docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-$BOX_NAME From 90d9eb715f477bbacdb85dd795ace63edad9120f Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 21:02:38 +0000 Subject: [PATCH 5/7] fix: independent repos --- yarn-project/boxes/run_tests | 38 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/yarn-project/boxes/run_tests b/yarn-project/boxes/run_tests index 6fc24364a0fd..c1b3bce05dfb 100755 --- a/yarn-project/boxes/run_tests +++ b/yarn-project/boxes/run_tests @@ -9,31 +9,43 @@ BOX_NAME=${1:-boxes-blank} cd $CURRENT_DIR/$BOX_NAME -export COMPOSE_FILE=${1:-docker-compose.yml} +export COMPOSE_FILE=${2:-docker-compose.yml} ecr_login -REPO=aztec-sandbox +SANDBOX=aztec-sandbox +YARN_PROJECT=yarn-project -ensure_repo $REPO $ECR_REGION refresh_lifecycle -CONTENT_HASH=$(calculate_content_hash $REPO) -BASE_TAG=cache-$CONTENT_HASH +SANDBOX_IMAGE_URI=$(calculate_image_uri $SANDBOX) +YP_IMAGE_URI=$(calculate_image_uri $YARN_PROJECT) + +ensure_repo $SANDBOX $ECR_REGION refresh_lifecycle +CONTENT_HASH=$(calculate_content_hash $SANDBOX) +BASE_TAG=cache-$CONTENT_HASH-$BOX_NAME # Append the box name to seperate the tag SUCCESS_TAG=$BASE_TAG echo "Content hash: $CONTENT_HASH" -if check_rebuild $SUCCESS_TAG $REPO; then +if check_rebuild $SUCCESS_TAG $SANDBOX; then echo "No rebuild required." + retry tag_remote_image $SANDBOX $BASE_TAG $SUCCESS_TAG exit 0 fi -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)-x86_64" -retry docker pull $(calculate_image_uri $REPO)-x86_64 -retry docker tag $(calculate_image_uri $REPO)-x86_64 aztecprotocol/$REPO:latest -REPO=yarn-project -echo "pulling docker image for $REPO $(calculate_image_uri $REPO)" -retry docker pull $(calculate_image_uri $REPO) -retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest +echo "pulling docker image for $SANDBOX $SANDBOX_IMAGE_URI-x86_64" +retry docker pull $SANDBOX_IMAGE_URI-x86_64 +retry docker tag $SANDBOX_IMAGE_URI-x86_64 aztecprotocol/$SANDBOX:latest + +echo "pulling docker image for $YARN_PROJECT $YP_IMAGE_URI" +retry docker pull $YP_IMAGE_URI +retry docker tag $YP_IMAGE_URI aztecprotocol/$YARN_PROJECT:latest docker-compose rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-$BOX_NAME + + + +# Success - push a new tag for the commit hash. +IMAGE_COMMIT_URI=$SANDBOX_IMAGE_URI-$BOX_NAME +retry docker tag $SANDBOX_IMAGE_URI-x86_64 $IMAGE_COMMIT_URI +retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 \ No newline at end of file From 0cba8a400e8dfca93ad9777c3933bd2099c6c762 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:21:49 +0000 Subject: [PATCH 6/7] Revert "pls rebuild" This reverts commit a31bfe1a0f0dec5b5c27c2676e792e38643e90e6. --- yarn-project/yarn-project-base/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index f5b60bbda1a2..4d92493bd06f 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -46,7 +46,6 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts FROM node:18-alpine RUN apk update && apk add --no-cache bash jq curl - WORKDIR /usr/src/yarn-project # The dockerignore file ensures the context only contains package.json and tsconfig.json files. From 0f2bc2054ff558dab3efe955a085a8dbdf4ae58c Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:28:50 +0000 Subject: [PATCH 7/7] chore: cleanup --- yarn-project/boxes/run_tests | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/yarn-project/boxes/run_tests b/yarn-project/boxes/run_tests index c1b3bce05dfb..80ee1ed87e6c 100755 --- a/yarn-project/boxes/run_tests +++ b/yarn-project/boxes/run_tests @@ -4,6 +4,8 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu +# The box name is the name of the directory containing the docker-compose.yml file +# The current dir is assumed to be `yarn-project/boxes`, as this script `yarn-project/boxes/run_tests` CURRENT_DIR=`dirname $0` BOX_NAME=${1:-boxes-blank} @@ -13,12 +15,17 @@ export COMPOSE_FILE=${2:-docker-compose.yml} ecr_login +# There are two dependent docker images, sandbox and yarn-project +# We must pull their ecr images, then retag them in ci as local images for our docker compose to work. SANDBOX=aztec-sandbox YARN_PROJECT=yarn-project SANDBOX_IMAGE_URI=$(calculate_image_uri $SANDBOX) YP_IMAGE_URI=$(calculate_image_uri $YARN_PROJECT) +################### CHECK REBUILD ################### +# Before doing any work we check if we need to rebuild +# Each box is suffixed with the box name, such that rebuilding of each box is independent, if one fails we don't rebuild the others ensure_repo $SANDBOX $ECR_REGION refresh_lifecycle CONTENT_HASH=$(calculate_content_hash $SANDBOX) BASE_TAG=cache-$CONTENT_HASH-$BOX_NAME # Append the box name to seperate the tag @@ -31,21 +38,26 @@ if check_rebuild $SUCCESS_TAG $SANDBOX; then exit 0 fi +################### PREAMBLE ################### -echo "pulling docker image for $SANDBOX $SANDBOX_IMAGE_URI-x86_64" -retry docker pull $SANDBOX_IMAGE_URI-x86_64 -retry docker tag $SANDBOX_IMAGE_URI-x86_64 aztecprotocol/$SANDBOX:latest +# Pull images from ecr and retag for the docker compsoe +SANDBOX_IMAGE=$SANDBOX_IMAGE_URI-x86_64 +echo "pulling docker image for $SANDBOX $SANDBOX_IMAGE" +retry docker pull $SANDBOX_IMAGE +retry docker tag $SANDBOX_IMAGE aztecprotocol/$SANDBOX:latest echo "pulling docker image for $YARN_PROJECT $YP_IMAGE_URI" retry docker pull $YP_IMAGE_URI retry docker tag $YP_IMAGE_URI aztecprotocol/$YARN_PROJECT:latest +################### RUN TESTS ################### + docker-compose rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from boxes-$BOX_NAME +################### POST TEST ################### - -# Success - push a new tag for the commit hash. +# Success - push a new tag for the commit hash with the box name appended IMAGE_COMMIT_URI=$SANDBOX_IMAGE_URI-$BOX_NAME -retry docker tag $SANDBOX_IMAGE_URI-x86_64 $IMAGE_COMMIT_URI +retry docker tag $SANDBOX_IMAGE $IMAGE_COMMIT_URI retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 \ No newline at end of file