From 02a30cdb5a00a3ea09632bcfab9e83d56441e821 Mon Sep 17 00:00:00 2001 From: Kiran Parajuli Date: Thu, 25 Aug 2022 14:31:40 +0545 Subject: [PATCH 1/4] use ocis cache from cache used by multiple repos Signed-off-by: Kiran Parajuli --- .drone.star | 181 +++++++++++-------- tests/drone/build-ocis.sh | 32 ---- tests/drone/check-for-existing-oCIS-cache.sh | 2 +- 3 files changed, 102 insertions(+), 113 deletions(-) delete mode 100755 tests/drone/build-ocis.sh diff --git a/.drone.star b/.drone.star index 18300c1e867..0162c48fdda 100644 --- a/.drone.star +++ b/.drone.star @@ -6,7 +6,7 @@ NOTIFICATIONS = 3 ALPINE_GIT = "alpine/git:latest" DEEPDRIVER_DOCKER_ORACLE_XE_11G = "deepdiver/docker-oracle-xe-11g:latest" DRONE_CLI_ALPINE = "drone/cli:alpine" -MINIO_MC = "minio/mc:RELEASE.2021-03-23T05-46-11Z" +MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z" OC_CI_ALPINE = "owncloudci/alpine:latest" OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier" OC_CI_CORE_NODEJS = "owncloudci/core:nodejs14" @@ -41,6 +41,7 @@ dir = { "federated": "/var/www/owncloud/federated", "server": "/var/www/owncloud/server", "web": "/var/www/owncloud/web", + "ocis": "/var/www/owncloud/ocis-build" } config = { @@ -705,6 +706,38 @@ ocisSpecificTestSuites = [ "webUIUserJourney", ] +# minio mc environment variables +minio_mc_environment = { + "CACHE_BUCKET": { + "from_secret": "cache_public_s3_bucket", + }, + "MC_HOST": { + "from_secret": "cache_s3_endpoint", + }, + "AWS_ACCESS_KEY_ID": { + "from_secret": "cache_s3_access_key", + }, + "AWS_SECRET_ACCESS_KEY": { + "from_secret": "cache_s3_secret_key", + }, +} + +go_step_volumes = [{ + "name": "server", + "path": "/srv/app", +}, { + "name": "gopath", + "path": "/go", +}, { + "name": "configs", + "path": "/srv/config", +}] + +web_workspace = { + "base": dir["base"], + "path": config["app"] +} + def checkTestSuites(): for testGroupName, test in config["acceptance"].items(): suites = [] @@ -1189,7 +1222,7 @@ def e2eTests(ctx): restoreBuildArtifactCache(ctx, "web-dist", "dist") + \ installYarn() + \ setupServerConfigureWeb(logLevel) + \ - getOcis() + \ + restoreOcisCache() + \ ocisService() + \ getSkeletonFiles() + \ copyFilesForUpload() + \ @@ -1340,7 +1373,7 @@ def acceptance(ctx): if (params["runningOnOCIS"]): # Services and steps required for running tests with oCIS - steps += getOcis() + ocisService() + getSkeletonFiles() + steps += restoreOcisCache() + ocisService() + getSkeletonFiles() else: # Services and steps required for running tests with oc10 @@ -2156,7 +2189,7 @@ def ocisService(): "FRONTEND_SEARCH_MIN_LENGTH": "2", }, "commands": [ - "cd %s/ocis-build" % dir["base"], + "cd %s" % dir["ocis"], "mkdir -p /srv/app/tmp/ocis/owncloud/data/", "mkdir -p /srv/app/tmp/ocis/storage/users/", "./ocis init", @@ -2221,6 +2254,23 @@ def ocisWebService(): }], }] +def checkForExistingOcisCache(ctx): + sdk_repo_path = "https://raw.githubusercontent.com/owncloud/web/%s" % ctx.build.commit + return [ + { + "name": "check-for-exisiting-cache", + "image": OC_UBUNTU, + "environment": minio_mc_environment, + "commands": [ + "curl -o .drone.env %s/.drone.env" % sdk_repo_path, + "curl -o check-oCIS-cache.sh %s/tests/drone/check-oCIS-cache.sh" % sdk_repo_path, + ". ./.drone.env", + "bash check-oCIS-cache.sh", + "pwd && ls -la", + ], + }, + ] + def setupServerConfigureWeb(logLevel): return [{ "name": "setup-server-configure-web", @@ -2385,11 +2435,12 @@ def cacheOcisPipeline(ctx): "kind": "pipeline", "type": "docker", "name": "cache-ocis", - "workspace": { - "base": dir["base"], - "path": config["app"], + "workspace": web_workspace, + "clone": { + "disable": True, }, - "steps": buildOCISCache() + + "steps": checkForExistingOcisCache(ctx) + + buildOcis() + cacheOcis() + listRemoteCache(), "volumes": [{ @@ -2405,112 +2456,82 @@ def cacheOcisPipeline(ctx): }, }] -def getOcis(): +def restoreOcisCache(): return [{ - "name": "get-ocis-from-cache", + "name": "restore-ocis-cache", "image": MINIO_MC, - "failure": "ignore", - "environment": { - "MC_HOST": { - "from_secret": "cache_s3_endpoint", - }, - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_s3_secret_key", - }, - }, + "environment": minio_mc_environment, "commands": [ - "source %s/.drone.env" % dir["web"], - "mkdir -p %s/ocis-build" % dir["base"], + ". ./.drone.env", + "rm -rf %s" % dir["ocis"], + "mkdir -p %s" % dir["ocis"], "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc mirror s3/owncloud/web/ocis-build/$OCIS_COMMITID %s/ocis-build/" % dir["base"], - "chmod +x %s/ocis-build/ocis" % dir["base"], + "mc cp -r -a s3/$CACHE_BUCKET/ocis-build/$OCIS_COMMITID/ocis %s" % dir["ocis"], ], }] -def buildOCISCache(): +def buildOcis(): + ocis_repo_url = "https://github.com/owncloud/ocis.git" return [ { - "name": "check-for-exisiting-cache", - "image": OC_UBUNTU, + "name": "clone-ocis", + "image": OC_CI_GOLANG, "commands": [ - "bash ./tests/drone/check-for-existing-oCIS-cache.sh", + "source .drone.env", + "cd $GOPATH/src", + "mkdir -p github.com/owncloud", + "cd github.com/owncloud", + "git clone -b $OCIS_BRANCH --single-branch %s" % ocis_repo_url, + "cd ocis", + "git checkout $OCIS_COMMITID", ], + "volumes": go_step_volumes, }, { "name": "generate-ocis", "image": OC_CI_NODEJS, - "environment": { - "GOPATH": "/go", - }, "commands": [ - "./tests/drone/build-ocis.sh nodejs", - ], - "volumes": [ - { - "name": "gopath", - "path": "/go", - }, + # we cannot use the $GOPATH here because of different base image + "cd /go/src/github.com/owncloud/ocis/", + "retry -t 3 'make ci-node-generate'", ], + "volumes": go_step_volumes, }, { "name": "build-ocis", "image": OC_CI_GOLANG, "commands": [ - "./tests/drone/build-ocis.sh golang", - ], - "volumes": [ - { - "name": "gopath", - "path": "/go", - }, + "source .drone.env", + "cd $GOPATH/src/github.com/owncloud/ocis/ocis", + "retry -t 3 'make build'", + "mkdir -p %s/$OCIS_COMMITID" % dir["base"], + "cp bin/ocis %s/$OCIS_COMMITID" % dir["base"], ], + "volumes": go_step_volumes, }, ] def cacheOcis(): return [{ - "name": "upload-ocis-bin", - "image": PLUGINS_S3, - "pull": "if-not-exists", - "settings": { - "bucket": "owncloud", - "endpoint": { - "from_secret": "cache_s3_endpoint", - }, - "path_style": True, - "source": "%s/ocis-build/**/*" % dir["base"], - "strip_prefix": "%s/ocis-build" % dir["base"], - "target": "/web/ocis-build/", - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - }, + "name": "upload-ocis-cache", + "image": MINIO_MC, + "environment": minio_mc_environment, + "commands": [ + ". ./.drone.env", + "pwd && ls -la %s/$OCIS_COMMITID" % dir["base"], + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc cp -r -a %s/$OCIS_COMMITID/ocis s3/$CACHE_BUCKET/ocis-build/$OCIS_COMMITID" % dir["base"], + ], }] def listRemoteCache(): return [{ - "name": "list-ocis-bin-cache", + "name": "list-remote-cache", "image": MINIO_MC, - "failure": "ignore", - "environment": { - "MC_HOST": { - "from_secret": "cache_s3_endpoint", - }, - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_s3_secret_key", - }, - }, + "environment": minio_mc_environment, "commands": [ - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY && mc find s3/owncloud/web/ocis-build", + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc ls --recursive s3/$CACHE_BUCKET/ocis-build", ], }] diff --git a/tests/drone/build-ocis.sh b/tests/drone/build-ocis.sh deleted file mode 100755 index 4c186a61c7a..00000000000 --- a/tests/drone/build-ocis.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# {string} which step to run -# options: nodejs, golang -STEP=$1 - -REPO_URL=https://github.com/owncloud/ocis.git -BASE_PATH=/var/www/owncloud/ocis-build/ - -source .drone.env - -if [ "$STEP" == "nodejs" ] -then - mkdir -p "$GOPATH"/src/github.com/owncloud/ - cd "$GOPATH"/src/github.com/owncloud/ || exit - git clone -b "$OCIS_BRANCH" --single-branch --no-tags $REPO_URL - cd ocis || exit - git checkout "$OCIS_COMMITID" - make ci-node-generate -else # golang - cd "$GOPATH"/src/github.com/owncloud/ocis/ocis || exit - if make build - then - echo "oCIS build successful." - else - echo "oCIS build failed." - exit 1 - fi - mkdir -p "$BASE_PATH""$OCIS_COMMITID" - cp bin/ocis "$BASE_PATH""$OCIS_COMMITID"/ - ls -la "$BASE_PATH""$OCIS_COMMITID"/ -fi diff --git a/tests/drone/check-for-existing-oCIS-cache.sh b/tests/drone/check-for-existing-oCIS-cache.sh index 2a90c337239..237074b82e9 100644 --- a/tests/drone/check-for-existing-oCIS-cache.sh +++ b/tests/drone/check-for-existing-oCIS-cache.sh @@ -2,7 +2,7 @@ source .drone.env -url="https://cache.owncloud.com/owncloud/web/ocis-build/$OCIS_COMMITID/ocis" +url="https://cache.owncloud.com/$CACHE_BUCKET/ocis-build/$OCIS_COMMITID/ocis" echo "Checking oCIS version - $OCIS_COMMITID in cache." echo "Downloading oCIS from '$url'." From 7039b1ea46dfcf39cd5f33309fff1a5c1daef96a Mon Sep 17 00:00:00 2001 From: Kiran Parajuli Date: Thu, 25 Aug 2022 14:36:58 +0545 Subject: [PATCH 2/4] Fixes Signed-off-by: Kiran Parajuli --- .drone.star | 11 +++++------ ...for-existing-oCIS-cache.sh => check-oCIS-cache.sh} | 0 2 files changed, 5 insertions(+), 6 deletions(-) rename tests/drone/{check-for-existing-oCIS-cache.sh => check-oCIS-cache.sh} (100%) diff --git a/.drone.star b/.drone.star index 0162c48fdda..2a6886afa95 100644 --- a/.drone.star +++ b/.drone.star @@ -41,7 +41,7 @@ dir = { "federated": "/var/www/owncloud/federated", "server": "/var/www/owncloud/server", "web": "/var/www/owncloud/web", - "ocis": "/var/www/owncloud/ocis-build" + "ocis": "/var/www/owncloud/ocis-build", } config = { @@ -735,7 +735,7 @@ go_step_volumes = [{ web_workspace = { "base": dir["base"], - "path": config["app"] + "path": config["app"], } def checkTestSuites(): @@ -2255,18 +2255,17 @@ def ocisWebService(): }] def checkForExistingOcisCache(ctx): - sdk_repo_path = "https://raw.githubusercontent.com/owncloud/web/%s" % ctx.build.commit + web_repo_path = "https://raw.githubusercontent.com/owncloud/web/%s" % ctx.build.commit return [ { "name": "check-for-exisiting-cache", "image": OC_UBUNTU, "environment": minio_mc_environment, "commands": [ - "curl -o .drone.env %s/.drone.env" % sdk_repo_path, - "curl -o check-oCIS-cache.sh %s/tests/drone/check-oCIS-cache.sh" % sdk_repo_path, + "curl -o .drone.env %s/.drone.env" % web_repo_path, + "curl -o check-oCIS-cache.sh %s/tests/drone/check-oCIS-cache.sh" % web_repo_path, ". ./.drone.env", "bash check-oCIS-cache.sh", - "pwd && ls -la", ], }, ] diff --git a/tests/drone/check-for-existing-oCIS-cache.sh b/tests/drone/check-oCIS-cache.sh similarity index 100% rename from tests/drone/check-for-existing-oCIS-cache.sh rename to tests/drone/check-oCIS-cache.sh From 29c0c3bc3d61150210e8543d46fdb7aa4140d83d Mon Sep 17 00:00:00 2001 From: Kiran Parajuli Date: Thu, 25 Aug 2022 16:53:37 +0545 Subject: [PATCH 3/4] Merge list and cache step Signed-off-by: Kiran Parajuli --- .drone.star | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.drone.star b/.drone.star index 2a6886afa95..53fb6abab04 100644 --- a/.drone.star +++ b/.drone.star @@ -2440,8 +2440,7 @@ def cacheOcisPipeline(ctx): }, "steps": checkForExistingOcisCache(ctx) + buildOcis() + - cacheOcis() + - listRemoteCache(), + cacheOcis(), "volumes": [{ "name": "gopath", "temp": {}, @@ -2520,16 +2519,6 @@ def cacheOcis(): "pwd && ls -la %s/$OCIS_COMMITID" % dir["base"], "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", "mc cp -r -a %s/$OCIS_COMMITID/ocis s3/$CACHE_BUCKET/ocis-build/$OCIS_COMMITID" % dir["base"], - ], - }] - -def listRemoteCache(): - return [{ - "name": "list-remote-cache", - "image": MINIO_MC, - "environment": minio_mc_environment, - "commands": [ - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", "mc ls --recursive s3/$CACHE_BUCKET/ocis-build", ], }] From e008a4b5263180b74510dc1eea2efd0fd63ccc06 Mon Sep 17 00:00:00 2001 From: Kiran Parajuli Date: Fri, 26 Aug 2022 13:13:52 +0545 Subject: [PATCH 4/4] Remove debug lines Signed-off-by: Kiran Parajuli --- .drone.star | 1 - 1 file changed, 1 deletion(-) diff --git a/.drone.star b/.drone.star index 53fb6abab04..1cd81e9faf9 100644 --- a/.drone.star +++ b/.drone.star @@ -2516,7 +2516,6 @@ def cacheOcis(): "environment": minio_mc_environment, "commands": [ ". ./.drone.env", - "pwd && ls -la %s/$OCIS_COMMITID" % dir["base"], "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", "mc cp -r -a %s/$OCIS_COMMITID/ocis s3/$CACHE_BUCKET/ocis-build/$OCIS_COMMITID" % dir["base"], "mc ls --recursive s3/$CACHE_BUCKET/ocis-build",