From e0a54a067cde4e9c49bfbfe6fc314d5e072b8c31 Mon Sep 17 00:00:00 2001
From: Jonas Kunz <jonas.kunz@elastic.co>
Date: Mon, 6 Feb 2023 16:01:53 +0100
Subject: [PATCH 01/29] Create GH actions workflow for agent release

---
 .ci/ReleaseChangelog.java      | 126 +++++++++++++++++++
 .github/workflows/release.yml  | 224 +++++++++++++++++++++++++++++++++
 scripts/jenkins/push_docker.sh |   4 +-
 3 files changed, 352 insertions(+), 2 deletions(-)
 create mode 100644 .ci/ReleaseChangelog.java
 create mode 100644 .github/workflows/release.yml

diff --git a/.ci/ReleaseChangelog.java b/.ci/ReleaseChangelog.java
new file mode 100644
index 0000000000..8d6092e0e1
--- /dev/null
+++ b/.ci/ReleaseChangelog.java
@@ -0,0 +1,126 @@
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.OptionalInt;
+import java.util.function.Predicate;
+
+public class ReleaseChangelog {
+
+    public static void main(String[] args) throws IOException {
+        if (args.length != 2) {
+            System.out.println("Expected exactly two arguments: <ChangelogFile> <VersionToRelease>");
+            System.exit(-1);
+        }
+        Path fileName = Paths.get(args[0]);
+        String version = args[1].trim();
+        if (!version.matches("\\d+\\.\\d+\\.\\d+")) {
+            System.out.println("Version must be in the format x.x.x but was not: " + version);
+            System.exit(-1);
+        }
+        Lines f = new Lines(Files.readAllLines(fileName, StandardCharsets.UTF_8));
+        int unreleasedStart = f.findLine(str -> str.trim().equals("=== Unreleased"), 0).orElseThrow() + 1;
+        int unreleasedEnd = f.findLine(str -> str.startsWith("[[release-notes-"), unreleasedStart).orElseThrow();
+
+        Lines changes = f.cut(unreleasedStart, unreleasedEnd);
+        f.insert(new Lines(List.of("")), unreleasedStart); //add a blank line below unreleased heading
+        changes.trim();
+
+        // a few sanity checks
+        if (changes.lineCount() == 0) {
+            System.out.println("Unreleased changes are empty, there must be something wrong!");
+            System.exit(-1);
+        }
+        OptionalInt wrongIndentedHeading = changes.findLine(str -> str.matches("^==?=?=?[^=].*"), 0);
+        if (wrongIndentedHeading.isPresent()) {
+            System.out.println("Found heading which is too high level (must be at least =====) within changes: "
+                + changes.getLine(wrongIndentedHeading.getAsInt()));
+            System.exit(-1);
+        }
+
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy/MM/dd")
+            .withZone(ZoneOffset.UTC);
+
+        changes.insert(new Lines(List.of(
+            "",
+            String.format("[[release-notes-%s]]", version),
+            String.format("==== %s - %s", version, formatter.format(Instant.now())),
+            ""
+        )), 0);
+
+        int majorVersion = Integer.parseInt(version.split("\\.")[0]);
+        String majorHeading = String.format("=== Java Agent version %d.x", majorVersion);
+        OptionalInt sectionStart = f.findLine(str -> str.trim().equals(majorHeading), 0);
+        if (sectionStart.isEmpty()) {
+            System.out.println("Could not find heading for major version: " + majorHeading);
+            System.out.println("Is this a new major version? If yes, please add an empty section for it manually to the Changelog");
+            System.exit(-1);
+        }
+
+        f.insert(changes, sectionStart.getAsInt() + 1);
+
+        Files.writeString(fileName, f.toString(), StandardCharsets.UTF_8);
+    }
+
+    static class Lines {
+
+        private final List<String> lines;
+
+        public Lines(List<String> lines) {
+            this.lines = new ArrayList<>(lines);
+        }
+
+        int lineCount() {
+            return lines.size();
+        }
+
+        OptionalInt findLine(Predicate<String> condition, int startAt) {
+            for (int i = startAt; i < lines.size(); i++) {
+                if (condition.test(lines.get(i))) {
+                    return OptionalInt.of(i);
+                }
+            }
+            return OptionalInt.empty();
+        }
+
+        Lines cut(int startInclusive, int endExclusive) {
+            List<String> cutLines = new ArrayList<>();
+            for (int i = startInclusive; i < endExclusive; i++) {
+                cutLines.add(lines.remove(startInclusive));
+            }
+            return new Lines(cutLines);
+        }
+
+        void insert(Lines other, int insertAt) {
+            this.lines.addAll(insertAt, other.lines);
+        }
+
+        /**
+         * Trims lines consisting of only blanks at the top and bottom
+         */
+        void trim() {
+            while (!lines.isEmpty() && lines.get(0).matches("\\s*")) {
+                lines.remove(0);
+            }
+            while (!lines.isEmpty() && lines.get(lines.size() - 1).matches("\\s*")) {
+                lines.remove(lines.size() - 1);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return String.join("\n", lines);
+        }
+
+        public String getLine(int number) {
+            return lines.get(number);
+        }
+    }
+
+}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000000..113a8557d2
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,224 @@
+---
+# Releases the agent
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'The branch to release'
+        required: true
+        default: 'main'
+      version:
+        description: 'The version to release (e.g. 1.2.3). This workflow will automatically perform the required version bumps'
+        required: true
+      update_changelog:
+        description: |
+          If enabled, everything in the changelog from the "Unreleased" section will be automatically moved to a new section for the new release.
+          If disabled, the changelog needs to be prepared for the release manually before triggering this workflow.
+        type: boolean
+        required: true
+        default: true
+
+env:
+  JAVA_VERSION: 17
+  JAVA_DIST: adopt
+  MAVEN_CONFIG: >-
+    -V
+    -B
+    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
+    -Dhttps.protocols=TLSv1.2 -Dmaven.wagon.http.retryHandler.count=3
+    -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+
+jobs:
+  prepare_release:
+    name: "Changelog and Version Bump"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        #TODO: Use apmmachine GH user + setup push authorization (SSH?)
+        with:
+          ref: ${{ inputs.branch }}
+      - name: Set up git user
+        run: |
+          git config user.name todo-github-actions
+          git config user.email todo-github-actions@github.com
+      - name: Set up JDK ${{ env.JAVA_VERSION }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_DIST }}
+          cache: 'maven'
+      - name: Prepare changelog for release
+        if: ${{ inputs.update_changelog }}
+        run: |
+          java .ci/ReleaseChangelog.java CHANGELOG.asciidoc ${{ inputs.version }}
+          git commit -m "Prepare changelog for release ${{ inputs.version }}" CHANGELOG.asciidoc
+      - name: Bump version and add git tag
+        run: ./mvnw release:prepare -B -DpushChanges=false "-Darguments=-DskipTests -Dmaven.javadoc.skip=true" -DreleaseVersion=${{ inputs.version }}
+      - run: git push --atomic origin ${{ inputs.branch }} v${{ inputs.version }}
+
+
+  maven_central_deploy:
+    name: "Deploy to Maven Central (Buildkite)"
+    runs-on: ubuntu-latest
+    needs:
+      - prepare_release
+
+    # TODO: Run the deployment on Buildkite
+    # Buildkite needs to
+    # - checkout the tag "v${{ inputs.version }}" (which is created by the prepare_release job)
+    # - run ./mvnw -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode
+    # Note that the previous ./mvnw release:prepare release:perform from jenkins are not required anymore:
+    # prepare_release does all the fiddling with the git history
+    steps:
+      - run: echo "Not implemented yet!"
+
+  await_artifact_on_maven_central:
+    name: "Wait for artifacts to be available on maven central"
+    runs-on: ubuntu-latest
+    needs:
+      - maven_central_deploy
+    steps:
+      - uses: actions/checkout@v3
+      - shell: bash
+        run: |
+          until .ci/release/wait_maven_artifact_published.sh ${{ inputs.version }}
+          do
+            echo "Artifacts not found on maven central. Sleeping 30 seconds, retrying afterwards"
+            sleep 30s
+          done
+  
+
+  update_major_branch:
+    name: "Update Major Branch"
+    runs-on: ubuntu-latest
+    needs:
+      - await_artifact_on_maven_central
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+        #TODO: Use apmmachine GH user + setup authorization (SSH?)
+      - run: .ci/release/update_major_branch.sh ${{ inputs.version }}
+      - run: git push -f origin "$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')"
+
+  update_cloudfoundry:
+    name: "Update Cloudfoundry"
+    runs-on: ubuntu-latest
+    needs:
+      - await_artifact_on_maven_central
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+        #TODO: Use apmmachine + setup authorization (SSH?)
+      - name: Set up git user
+        run: |
+          git config user.name todo-github-actions
+          git config user.email todo-github-actions@github.com
+      - name: "Update Cloudfoundry index.yml file"
+        shell: bash
+        run: .ci/release/update_cloudfoundry.sh ${{ inputs.version }}
+      - run: git push origin main
+
+
+  build_docker_images:
+    name: "Build and push docker images"
+    runs-on: ubuntu-latest
+    needs:
+      - await_artifact_on_maven_central
+    env:
+      TAG_NAME: v${{ inputs.version }}
+      SONATYPE_FALLBACK: 1
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+          fetch-depth: 0 # Load entire history as it is required for the push-script
+      - name: "Build docker image"
+        shell: bash
+        # TODO: add docker login
+        run: |
+          ./scripts/jenkins/build_docker.sh
+          ./scripts/jenkins/push_docker.sh
+
+  publish_aws_lambda:
+    name: "Publish AWS Lambda"
+    runs-on: ubuntu-latest
+    needs:
+      - await_artifact_on_maven_central
+    outputs:
+      arn_content: ${{ steps.arn_output.outputs.arn_content }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+      # TODO: actual lambda creation + upload
+      - name: Setup dummy ARN file
+        run: |
+          {
+              echo "### ARNs of the APM Java Agent's AWS Lambda Layer"
+              echo ''
+              echo '|Region|ARN|'
+              echo '|------|---|'
+          } > ".ci/arn-file.md"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: arn-file
+          path: .ci/arn-file.md
+      - name: Add ARN file to output
+        id: arn_output
+        run: |
+          echo 'arn_content<<ARN_CONTENT_EOF' >> $GITHUB_OUTPUT
+          cat .ci/arn-file.md >> $GITHUB_OUTPUT
+          echo 'ARN_CONTENT_EOF' >> $GITHUB_OUTPUT
+  
+
+  create_github_release:
+    name: "Create GitHub Release"
+    needs:
+      - publish_aws_lambda
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+      - name: Await release-notes published
+        shell: bash
+        run: |
+          until .ci/release/wait_release_notes_published.sh ${{ inputs.version }}
+          do
+            echo "Release notes not published yet. Sleeping 30 seconds, retrying afterwards"
+            sleep 30s
+          done
+      - name: Compute major.x branch
+        id: get_dotx_branch
+        run: echo "dotx_branch=$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')" >> $GITHUB_OUTPUT
+      - uses: ncipollo/release-action@v1
+        with:
+          #token: TODO: set apmmachine (or whoever releases) user token here!
+          tag: v${{ inputs.version }}
+          name: Release ${{ inputs.version }}
+          body: |
+            [Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
+            ${{ needs.publish_aws_lambda.outputs.arn_content }}
+
+  bump_opbeans:
+    name: "Bump Opbeans agent version"
+    runs-on: ubuntu-latest
+    steps:
+      #TODO: Use apmmachine + setup authorization (SSH?)
+      - uses: actions/checkout@v3
+        with:
+          repository: JonasKunz/opbeans-java
+          ref: main
+      - name: Set up git user
+        run: |
+          git config user.name todo-github-actions
+          git config user.email todo-github-actions@github.com
+      - name: Bump version
+        run: |
+          .ci/bump-version.sh ${{ inputs.version }}
+          git tag v${{ inputs.version }}
+      - run: git push --atomic origin main v${{ inputs.version }}
diff --git a/scripts/jenkins/push_docker.sh b/scripts/jenkins/push_docker.sh
index e6f26bd0ee..2953639ce9 100755
--- a/scripts/jenkins/push_docker.sh
+++ b/scripts/jenkins/push_docker.sh
@@ -48,7 +48,7 @@ if [ ${WORKERS+x} ]  # We are on a CI worker
 then
   retry $RETRIES docker push $DOCKER_PUSH_IMAGE || echo "Push failed after $RETRIES retries"
 else  # We are in a local (non-CI) environment
-  docker push $DOCKER_PUSH_IMAGE || echo "You may need to run 'docker login' first and then re-run this script"
+  docker push $DOCKER_PUSH_IMAGE || { echo "You may need to run 'docker login' first and then re-run this script"; exit 1; }
 fi
 
 readonly LATEST_TAG=$(git tag --list --sort=version:refname "v*" | grep -v RC | sed s/^v// | tail -n 1)
@@ -62,6 +62,6 @@ then
   then
     retry $RETRIES docker push $DOCKER_PUSH_IMAGE_LATEST || echo "Push failed after $RETRIES retries"
   else  # We are in a local (non-CI) environment
-    docker push $DOCKER_PUSH_IMAGE_LATEST || echo "You may need to run 'docker login' first and then re-run this script"
+    docker push $DOCKER_PUSH_IMAGE_LATEST || { echo "You may need to run 'docker login' first and then re-run this script"; exit 1; }
   fi
 fi

From 57e0a18df388aac8789b6ddb6833b625bfc2cbdd Mon Sep 17 00:00:00 2001
From: Jonas Kunz <jonas.kunz@elastic.co>
Date: Mon, 20 Feb 2023 12:40:50 +0100
Subject: [PATCH 02/29] Added missing job dependency

---
 .github/workflows/release.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 113a8557d2..d2c6522b82 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -179,6 +179,7 @@ jobs:
     name: "Create GitHub Release"
     needs:
       - publish_aws_lambda
+      - update_major_branch
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

From 9eb78abd3e62fc9709798a2023645385953e839a Mon Sep 17 00:00:00 2001
From: Jonas Kunz <jonas.kunz@elastic.co>
Date: Mon, 20 Feb 2023 12:47:51 +0100
Subject: [PATCH 03/29] Added another missing dependency

---
 .github/workflows/release.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d2c6522b82..c28d846292 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -208,6 +208,8 @@ jobs:
   bump_opbeans:
     name: "Bump Opbeans agent version"
     runs-on: ubuntu-latest
+    needs:
+      - await_artifact_on_maven_central
     steps:
       #TODO: Use apmmachine + setup authorization (SSH?)
       - uses: actions/checkout@v3

From 2414c74461f230a72982dcbc978648fb0a1db10f Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 8 May 2023 15:24:02 +0200
Subject: [PATCH 04/29] Add setup-git action and adapt permissions

---
 .github/workflows/release.yml | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c28d846292..8457bf7820 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,6 +1,6 @@
 ---
 # Releases the agent
-name: Release
+name: release
 
 on:
   workflow_dispatch:
@@ -30,19 +30,20 @@ env:
     -Dhttps.protocols=TLSv1.2 -Dmaven.wagon.http.retryHandler.count=3
     -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
 
+permissions:
+  contents: write
+
 jobs:
   prepare_release:
+    permissions:
+      contents: write
     name: "Changelog and Version Bump"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-        #TODO: Use apmmachine GH user + setup push authorization (SSH?)
         with:
           ref: ${{ inputs.branch }}
-      - name: Set up git user
-        run: |
-          git config user.name todo-github-actions
-          git config user.email todo-github-actions@github.com
+      - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: Set up JDK ${{ env.JAVA_VERSION }}
         uses: actions/setup-java@v3
         with:
@@ -99,7 +100,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: main
-        #TODO: Use apmmachine GH user + setup authorization (SSH?)
+      - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - run: .ci/release/update_major_branch.sh ${{ inputs.version }}
       - run: git push -f origin "$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')"
 
@@ -112,11 +113,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: main
-        #TODO: Use apmmachine + setup authorization (SSH?)
-      - name: Set up git user
-        run: |
-          git config user.name todo-github-actions
-          git config user.email todo-github-actions@github.com
+      - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: "Update Cloudfoundry index.yml file"
         shell: bash
         run: .ci/release/update_cloudfoundry.sh ${{ inputs.version }}
@@ -198,7 +195,6 @@ jobs:
         run: echo "dotx_branch=$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')" >> $GITHUB_OUTPUT
       - uses: ncipollo/release-action@v1
         with:
-          #token: TODO: set apmmachine (or whoever releases) user token here!
           tag: v${{ inputs.version }}
           name: Release ${{ inputs.version }}
           body: |
@@ -211,15 +207,11 @@ jobs:
     needs:
       - await_artifact_on_maven_central
     steps:
-      #TODO: Use apmmachine + setup authorization (SSH?)
       - uses: actions/checkout@v3
         with:
           repository: JonasKunz/opbeans-java
           ref: main
-      - name: Set up git user
-        run: |
-          git config user.name todo-github-actions
-          git config user.email todo-github-actions@github.com
+      - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: Bump version
         run: |
           .ci/bump-version.sh ${{ inputs.version }}

From 1a00ccbd08db92362e82a174369df963c87849d9 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 8 May 2023 16:19:36 +0200
Subject: [PATCH 05/29] Add docker login

---
 .github/workflows/release.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8457bf7820..2a709e2fa7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -133,9 +133,15 @@ jobs:
         with:
           ref: main
           fetch-depth: 0 # Load entire history as it is required for the push-script
+      - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current
+        with:
+          registry: docker.elastic.co
+          secret: secret/apm-team/ci/docker-registry/prod
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
       - name: "Build docker image"
         shell: bash
-        # TODO: add docker login
         run: |
           ./scripts/jenkins/build_docker.sh
           ./scripts/jenkins/push_docker.sh

From 4c1baec75379c792449876513a0a756df45ab988 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 15 May 2023 12:27:39 +0200
Subject: [PATCH 06/29] Replace hardcoded occurrences of main with
 inputs.branch

---
 .github/workflows/release.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2a709e2fa7..409226fa18 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -112,12 +112,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: main
+          ref: ${{ inputs.branch }}
       - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: "Update Cloudfoundry index.yml file"
         shell: bash
         run: .ci/release/update_cloudfoundry.sh ${{ inputs.version }}
-      - run: git push origin main
+      - run: git push origin ${{ inputs.branch }}
 
 
   build_docker_images:
@@ -131,7 +131,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: main
+          ref: ${{ inputs.branch }}
           fetch-depth: 0 # Load entire history as it is required for the push-script
       - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current
         with:
@@ -156,7 +156,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: main
+          ref: ${{ inputs.branch }}
       # TODO: actual lambda creation + upload
       - name: Setup dummy ARN file
         run: |
@@ -187,7 +187,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: main
+          ref: ${{ inputs.branch }}
       - name: Await release-notes published
         shell: bash
         run: |
@@ -216,10 +216,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           repository: JonasKunz/opbeans-java
-          ref: main
+          ref: ${{ inputs.branch }}
       - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: Bump version
         run: |
           .ci/bump-version.sh ${{ inputs.version }}
           git tag v${{ inputs.version }}
-      - run: git push --atomic origin main v${{ inputs.version }}
+      - run: git push --atomic origin ${{ inputs.branch }} v${{ inputs.version }}

From 95d27d94a244b1cfc3d5409ac253bb55c42650b6 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 15 May 2023 12:48:36 +0200
Subject: [PATCH 07/29] Use gh cli instead of custom action

---
 .github/workflows/release.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 409226fa18..ea2e162d6c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -199,13 +199,14 @@ jobs:
       - name: Compute major.x branch
         id: get_dotx_branch
         run: echo "dotx_branch=$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')" >> $GITHUB_OUTPUT
-      - uses: ncipollo/release-action@v1
-        with:
-          tag: v${{ inputs.version }}
-          name: Release ${{ inputs.version }}
-          body: |
-            [Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
-            ${{ needs.publish_aws_lambda.outputs.arn_content }}
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create v${{ inputs.version }} \
+            --title="Release ${{ inputs.version }}" \
+            --notes="[Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
+            ${{ needs.publish_aws_lambda.outputs.arn_content }}"
 
   bump_opbeans:
     name: "Bump Opbeans agent version"

From 9e74b86bd18fceb16dc56e5fae36072aae5aa141 Mon Sep 17 00:00:00 2001
From: Jan Calanog <nejcalanog@gmail.com>
Date: Wed, 31 May 2023 15:13:33 +0200
Subject: [PATCH 08/29] Update .github/workflows/release.yml

Co-authored-by: Jonas Kunz <Jonas.Kunz@gmx.net>
---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ea2e162d6c..c84fe4fc0d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -216,7 +216,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          repository: JonasKunz/opbeans-java
+          repository: elastic/opbeans-java
           ref: ${{ inputs.branch }}
       - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - name: Bump version

From dbef751cbdf860d26982058ad08945ff0d030872 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 31 May 2023 15:15:15 +0200
Subject: [PATCH 09/29] Remove bump_opbeans job

This will be migrated to a updatecli job
---
 .github/workflows/release.yml | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c84fe4fc0d..729d750d20 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -207,20 +207,3 @@ jobs:
             --title="Release ${{ inputs.version }}" \
             --notes="[Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
             ${{ needs.publish_aws_lambda.outputs.arn_content }}"
-
-  bump_opbeans:
-    name: "Bump Opbeans agent version"
-    runs-on: ubuntu-latest
-    needs:
-      - await_artifact_on_maven_central
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: elastic/opbeans-java
-          ref: ${{ inputs.branch }}
-      - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
-      - name: Bump version
-        run: |
-          .ci/bump-version.sh ${{ inputs.version }}
-          git tag v${{ inputs.version }}
-      - run: git push --atomic origin ${{ inputs.branch }} v${{ inputs.version }}

From b4d7652bb300f06135f6887362df67161647c72c Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 31 May 2023 15:17:13 +0200
Subject: [PATCH 10/29] Remove maven config env

---
 .github/workflows/release.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 729d750d20..43a7a43494 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -23,12 +23,6 @@ on:
 env:
   JAVA_VERSION: 17
   JAVA_DIST: adopt
-  MAVEN_CONFIG: >-
-    -V
-    -B
-    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-    -Dhttps.protocols=TLSv1.2 -Dmaven.wagon.http.retryHandler.count=3
-    -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
 
 permissions:
   contents: write

From 79e768ad853235a07c6c4a5814129276a9cb2153 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 31 May 2023 15:21:23 +0200
Subject: [PATCH 11/29] Use input instead of hard coded ref

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 43a7a43494..b64cbdcc61 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -93,7 +93,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: main
+          ref: ${{ inputs.branch }}
       - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - run: .ci/release/update_major_branch.sh ${{ inputs.version }}
       - run: git push -f origin "$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')"

From ffb0d164091bfff04b8755da2625264bc8712fd7 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 31 May 2023 23:12:16 +0200
Subject: [PATCH 12/29] Add buildkite trigger action and complete lambda
 publish steps

---
 .buildkite/release.yml        | 14 +++++++++
 .ci/release.sh                | 33 ++++++++++++++++++++
 .github/workflows/release.yml | 57 +++++++++++++++++++++++------------
 3 files changed, 85 insertions(+), 19 deletions(-)
 create mode 100644 .buildkite/release.yml
 create mode 100755 .ci/release.sh

diff --git a/.buildkite/release.yml b/.buildkite/release.yml
new file mode 100644
index 0000000000..7f51edc121
--- /dev/null
+++ b/.buildkite/release.yml
@@ -0,0 +1,14 @@
+agents:
+  provider: "gcp"
+
+steps:
+  - label: "Run the snapshot"
+    key: "release"
+    commands: .ci/release.sh
+    artifact_paths:
+      - "release.txt"
+      - "**/target/*"
+
+notify:
+  - slack: "#apm-agent-java"
+    if: 'build.state != "passed"'
diff --git a/.ci/release.sh b/.ci/release.sh
new file mode 100755
index 0000000000..f8e5dd73d4
--- /dev/null
+++ b/.ci/release.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+##  This script runs the release given the different environment variables
+##    dry_run
+##
+##  It relies on the .buildkite/hooks/pre-command so the Vault and other tooling
+##  are prepared automatically by buildkite.
+##
+
+set -eo pipefail
+
+# Make sure we delete this folder before leaving even in case of failure
+clean_up () {
+  ARG=$?
+  export VAULT_TOKEN=$PREVIOUS_VAULT_TOKEN
+  echo "--- Deleting tmp workspace"
+  rm -rf $TMP_WORKSPACE
+  exit $ARG
+}
+trap clean_up EXIT
+
+echo "--- Debug JDK installation :coffee:"
+echo $JAVA_HOME
+echo $PATH
+java -version
+
+set +x
+echo "--- Deploy the release :package:"
+if [[ "$dry_run" == "true" ]] ; then
+  ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true | tee release.txt
+else
+  echo "hello" | tee release.txt
+  # ./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt
+fi
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b64cbdcc61..09f0e36b2f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,6 +19,10 @@ on:
         type: boolean
         required: true
         default: true
+      dry_run:
+        description: If set, run a dry-run release
+        default: false
+        type: boolean
 
 env:
   JAVA_VERSION: 17
@@ -59,15 +63,23 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - prepare_release
-
-    # TODO: Run the deployment on Buildkite
-    # Buildkite needs to
-    # - checkout the tag "v${{ inputs.version }}" (which is created by the prepare_release job)
-    # - run ./mvnw -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode
-    # Note that the previous ./mvnw release:prepare release:perform from jenkins are not required anymore:
-    # prepare_release does all the fiddling with the git history
     steps:
-      - run: echo "Not implemented yet!"
+      - id: buildkite
+        name: Run Deploy
+        uses: reakaleek/apm-pipeline-library/.github/actions/buildkite@feature/buildkite-download-artifacts
+        with:
+          vaultUrl: ${{ secrets.VAULT_ADDR }}
+          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
+          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
+          pipeline: apm-agent-java-release
+          pipelineVersion: ${{ inputs.version }}
+          waitFor: true
+          printBuildLogs: false
+          artifactName: lambda-zip
+          artifactPath: "elastic-apm-agent/target/elastic-apm-java-aws-lambda-layer-*.zip"
+          buildEnvVars: |
+            dry_run=${{ inputs.dry_run || 'false' }}
+
 
   await_artifact_on_maven_central:
     name: "Wait for artifacts to be available on maven central"
@@ -83,7 +95,6 @@ jobs:
             echo "Artifacts not found on maven central. Sleeping 30 seconds, retrying afterwards"
             sleep 30s
           done
-  
 
   update_major_branch:
     name: "Update Major Branch"
@@ -151,15 +162,23 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.branch }}
-      # TODO: actual lambda creation + upload
-      - name: Setup dummy ARN file
-        run: |
-          {
-              echo "### ARNs of the APM Java Agent's AWS Lambda Layer"
-              echo ''
-              echo '|Region|ARN|'
-              echo '|------|---|'
-          } > ".ci/arn-file.md"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lambda-zip
+          path: elastic-apm-agent/target
+      - uses: hashicorp/vault-action@v2.4.2
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          secrets: |
+            secret/observability-team/ci/service-account/apm-aws-lambda access_key_id | AWS_ACCESS_KEY_ID ;
+            secret/observability-team/ci/service-account/apm-aws-lambda secret_access_key | AWS_SECRET_ACCESS_KEY
+      - name: Publish
+        run: make -C .ci publish-in-all-aws-regions
+      - name: Create ARN file
+        run: make -C .ci create-arn-file
       - uses: actions/upload-artifact@v3
         with:
           name: arn-file
@@ -170,7 +189,7 @@ jobs:
           echo 'arn_content<<ARN_CONTENT_EOF' >> $GITHUB_OUTPUT
           cat .ci/arn-file.md >> $GITHUB_OUTPUT
           echo 'ARN_CONTENT_EOF' >> $GITHUB_OUTPUT
-  
+
 
   create_github_release:
     name: "Create GitHub Release"

From 57d965cd2514f2fe195c0350478af294da763a32 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 14:12:42 +0200
Subject: [PATCH 13/29] Cleanup

---
 .ci/release.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/release.sh b/.ci/release.sh
index f8e5dd73d4..dd5a516911 100755
--- a/.ci/release.sh
+++ b/.ci/release.sh
@@ -26,8 +26,8 @@ java -version
 set +x
 echo "--- Deploy the release :package:"
 if [[ "$dry_run" == "true" ]] ; then
+  # Build artifacts only
   ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true | tee release.txt
 else
-  echo "hello" | tee release.txt
-  # ./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt
+  ./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt
 fi

From d36f3237f5039d5dd46c9fc573941499606fe32a Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 15:57:39 +0200
Subject: [PATCH 14/29] Checkout tag after it's created by prepare_release

---
 .github/workflows/release.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 09f0e36b2f..3bb7141262 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -27,6 +27,7 @@ on:
 env:
   JAVA_VERSION: 17
   JAVA_DIST: adopt
+  TAG_NAME: v${{ inputs.version }}
 
 permissions:
   contents: write
@@ -55,7 +56,7 @@ jobs:
           git commit -m "Prepare changelog for release ${{ inputs.version }}" CHANGELOG.asciidoc
       - name: Bump version and add git tag
         run: ./mvnw release:prepare -B -DpushChanges=false "-Darguments=-DskipTests -Dmaven.javadoc.skip=true" -DreleaseVersion=${{ inputs.version }}
-      - run: git push --atomic origin ${{ inputs.branch }} v${{ inputs.version }}
+      - run: git push --atomic origin ${{ inputs.branch }} ${{ env.TAG_NAME }}
 
 
   maven_central_deploy:
@@ -72,7 +73,7 @@ jobs:
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
           vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
           pipeline: apm-agent-java-release
-          pipelineVersion: ${{ inputs.version }}
+          pipelineVersion: ${{ env.TAG_NAME }}
           waitFor: true
           printBuildLogs: false
           artifactName: lambda-zip
@@ -104,7 +105,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ inputs.branch }}
+          ref: ${{ env.TAG_NAME }}
       - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
       - run: .ci/release/update_major_branch.sh ${{ inputs.version }}
       - run: git push -f origin "$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')"
@@ -131,7 +132,6 @@ jobs:
     needs:
       - await_artifact_on_maven_central
     env:
-      TAG_NAME: v${{ inputs.version }}
       SONATYPE_FALLBACK: 1
     steps:
       - uses: actions/checkout@v3
@@ -200,7 +200,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ inputs.branch }}
+          ref: ${{ env.TAG_NAME }}
       - name: Await release-notes published
         shell: bash
         run: |
@@ -216,7 +216,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release create v${{ inputs.version }} \
+          gh release create ${{ env.TAG_NAME }} \
             --title="Release ${{ inputs.version }}" \
             --notes="[Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
             ${{ needs.publish_aws_lambda.outputs.arn_content }}"

From 4791c32eef75ee86e43e5de36b63c4d5f3266c7f Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 16:13:17 +0200
Subject: [PATCH 15/29] Rename scripts/jenkins folder to scripts/docker-release

---
 .ci/release/Jenkinsfile                          |  6 +++---
 .github/workflows/release.yml                    |  4 ++--
 CONTRIBUTING.md                                  | 16 ++++++++--------
 Jenkinsfile                                      |  2 +-
 .../{jenkins => docker-release}/build_docker.sh  |  0
 .../{jenkins => docker-release}/check_maven.sh   |  0
 .../fetch_nexus_id.sh                            |  0
 .../{jenkins => docker-release}/push_docker.sh   |  0
 .../run-benchmarks.sh                            |  0
 9 files changed, 14 insertions(+), 14 deletions(-)
 rename scripts/{jenkins => docker-release}/build_docker.sh (100%)
 rename scripts/{jenkins => docker-release}/check_maven.sh (100%)
 rename scripts/{jenkins => docker-release}/fetch_nexus_id.sh (100%)
 rename scripts/{jenkins => docker-release}/push_docker.sh (100%)
 rename scripts/{jenkins => docker-release}/run-benchmarks.sh (100%)

diff --git a/.ci/release/Jenkinsfile b/.ci/release/Jenkinsfile
index c2883860a4..99267a1b9f 100644
--- a/.ci/release/Jenkinsfile
+++ b/.ci/release/Jenkinsfile
@@ -97,7 +97,7 @@ pipeline {
             // If this fails, an exception should be thrown and execution will halt
             dir("${BASE_DIR}"){
               script {
-                def r = sh(label: "Check Maven status", script: "./scripts/jenkins/check_maven.sh -u https://status.maven.org/api/v2/summary.json --component OSSRH", returnStatus: true)
+                def r = sh(label: "Check Maven status", script: "./scripts/docker-release/check_maven.sh -u https://status.maven.org/api/v2/summary.json --component OSSRH", returnStatus: true)
                 if (r == 1) {
                   error("Failing release build because Maven is the OSSRH component is not fully operational. See https://status.maven.org/ for more details.")
                 }
@@ -264,10 +264,10 @@ pipeline {
             dir("${BASE_DIR}"){
               // fetch agent artifact from remote repository
               withEnv(["SONATYPE_FALLBACK=1"]) {
-                sh(label: "Build Docker image", script: "./scripts/jenkins/build_docker.sh")
+                sh(label: "Build Docker image", script: "./scripts/docker-release/build_docker.sh")
                   // Get Docker registry credentials
                   dockerLogin(secret: "${ELASTIC_DOCKER_SECRET}", registry: 'docker.elastic.co', role_id: 'apm-vault-role-id', secret_id: 'apm-vault-secret-id')
-                  sh(label: "Push Docker image", script: "./scripts/jenkins/push_docker.sh")
+                  sh(label: "Push Docker image", script: "./scripts/docker-release/push_docker.sh")
               }
             }
           }
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 3bb7141262..1fcd1267dd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -148,8 +148,8 @@ jobs:
       - name: "Build docker image"
         shell: bash
         run: |
-          ./scripts/jenkins/build_docker.sh
-          ./scripts/jenkins/push_docker.sh
+          ./scripts/docker-release/build_docker.sh
+          ./scripts/docker-release/push_docker.sh
 
   publish_aws_lambda:
     name: "Publish AWS Lambda"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 468d2ce4a0..8707aee2de 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -315,7 +315,7 @@ For illustration purpose, `1.2.3` will be the target release version, and the gi
    1. Download `elastic-apm-java-aws-lambda-layer-<VERSION>.zip` from the CI release job artifacts and upload it to the release draft
 1. Wait for released package to be available in [maven central](https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/)
 1. Build and push a Docker image using the instructions below
-   Use `SONATYPE_FALLBACK=1 scripts/jenkins/build_docker.sh` to build image with released artifact.
+   Use `SONATYPE_FALLBACK=1 scripts/docker-release/build_docker.sh` to build image with released artifact.
    Requires credentials, thus need to delegate this manual step to someone that has them.
 1. Update [`cloudfoundry/index.yml`](cloudfoundry/index.yml) on `main`.
 1. Publish release on Github. This will notify users watching repository.
@@ -336,7 +336,7 @@ docker pull  docker.elastic.co/observability/apm-agent-java:1.12.0
 #### Creating images for a Release
 
 Images are normally created and published as a part of the release process.
-Scripts which manage the Docker release process are located in [`scripts/jenkins/`](scripts/jenkins/).
+Scripts which manage the Docker release process are located in [`scripts/docker-release/`](scripts/docker-release/).
 
 ##### Building images locally
 
@@ -347,13 +347,13 @@ Building images on a workstation requires the following:
 * A local checkout of the `apm-agent-java` repo
 * The `git` command-line tool
 
-Local image building is handled via the [`build_docker.sh script`](scripts/jenkins/build_docker.sh).
+Local image building is handled via the [`build_docker.sh script`](scripts/docker-release/build_docker.sh).
 
 If you wish to use a locally built artifact in the built image, execute [`./mvnw package`](mvnw)`
 and ensure that artifacts are present in `elastic-apm-agent/target/*.jar`.
 
 To create a Docker image from artifacts generated by [`./mvnw package`](mvnw),
-run [`scripts/jenkins/build_docker.sh`](scripts/jenkins/build_docker.sh).
+run [`scripts/docker-release/build_docker.sh`](scripts/docker-release/build_docker.sh).
 
 Alternatively, it is also possible to use the most recent artifact from the [Sonatype
 repository](https://oss.sonatype.org/#nexus-search;gav~co.elastic.apm~apm-agent-java~~~).
@@ -361,16 +361,16 @@ repository](https://oss.sonatype.org/#nexus-search;gav~co.elastic.apm~apm-agent-
 To do so, first clean any artifacts with [`./mvnw clean`](mvnw) and then run the Docker
 build script with the `SONATYPE_FALLBACK` environment variable present. For example,
 
-`SONATYPE_FALLBACK=1 scripts/jenkins/build_docker.sh`
+`SONATYPE_FALLBACK=1 scripts/docker-release/build_docker.sh`
 
-After running the [`build_docker.sh`](scripts/jenkins/build_docker.sh) script, images can be seen by executing
+After running the [`build_docker.sh`](scripts/docker-release/build_docker.sh) script, images can be seen by executing
 `docker images|egrep docker.elastic.co/observability/apm-agent-java` which should
 produce output similar to the following:
 
 `docker.elastic.co/observability/apm-agent-java   1.12.0               1f45b5858d81        26 hours ago        10.6MB`
 
 No output from the above command indicates that the image did not build correctly
-and that the output of the [`build_docker.sh`](scripts/jenkins/build_docker.sh)
+and that the output of the [`build_docker.sh`](scripts/docker-release/build_docker.sh)
 script should be examined to determine the cause.
 
 
@@ -381,5 +381,5 @@ _Notice:_ You must have access to release secrets in order to push images.
 Prior to pushing images, you must login to the Elastic Docker repo using the correct
 credentials using the [`docker login`](https://docs.docker.com/engine/reference/commandline/login/) command.
 
-To push an image, run the [`scripts/jenkins/push_docker.sh`](scripts/jenkins/push_docker.sh)
+To push an image, run the [`scripts/docker-release/push_docker.sh`](scripts/docker-release/push_docker.sh)
 script. An image will be pushed.
diff --git a/Jenkinsfile b/Jenkinsfile
index 8d494125ef..313b73bea9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -131,7 +131,7 @@ pipeline {
               unstashV2(name: 'build', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
               dir("${BASE_DIR}"){
                 withOtelEnv() {
-                  sh './scripts/jenkins/run-benchmarks.sh'
+                  sh './scripts/docker-release/run-benchmarks.sh'
                 }
               }
             }
diff --git a/scripts/jenkins/build_docker.sh b/scripts/docker-release/build_docker.sh
similarity index 100%
rename from scripts/jenkins/build_docker.sh
rename to scripts/docker-release/build_docker.sh
diff --git a/scripts/jenkins/check_maven.sh b/scripts/docker-release/check_maven.sh
similarity index 100%
rename from scripts/jenkins/check_maven.sh
rename to scripts/docker-release/check_maven.sh
diff --git a/scripts/jenkins/fetch_nexus_id.sh b/scripts/docker-release/fetch_nexus_id.sh
similarity index 100%
rename from scripts/jenkins/fetch_nexus_id.sh
rename to scripts/docker-release/fetch_nexus_id.sh
diff --git a/scripts/jenkins/push_docker.sh b/scripts/docker-release/push_docker.sh
similarity index 100%
rename from scripts/jenkins/push_docker.sh
rename to scripts/docker-release/push_docker.sh
diff --git a/scripts/jenkins/run-benchmarks.sh b/scripts/docker-release/run-benchmarks.sh
similarity index 100%
rename from scripts/jenkins/run-benchmarks.sh
rename to scripts/docker-release/run-benchmarks.sh

From 9c6fc83dceb5a125c5e362da71f42fea811764cd Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 16:19:39 +0200
Subject: [PATCH 16/29] Checkout newly created tag instead of main

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1fcd1267dd..d8e66f95db 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -136,7 +136,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ inputs.branch }}
+          ref: ${{ env.TAG_NAME }}
           fetch-depth: 0 # Load entire history as it is required for the push-script
       - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current
         with:
@@ -161,7 +161,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: ${{ inputs.branch }}
+          ref: ${{ env.TAG_NAME }}
       - uses: actions/download-artifact@v3
         with:
           name: lambda-zip

From 1bdef7e1d3a213d6345ebd5fac4f6aa18869bab4 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 17:20:25 +0200
Subject: [PATCH 17/29] Add timeout-minutes to until loops

---
 .github/workflows/release.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d8e66f95db..f342146e30 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -89,7 +89,9 @@ jobs:
       - maven_central_deploy
     steps:
       - uses: actions/checkout@v3
-      - shell: bash
+      - name: Await artifacts published in maven central
+        shell: bash
+        timeout-minutes: 120
         run: |
           until .ci/release/wait_maven_artifact_published.sh ${{ inputs.version }}
           do
@@ -203,6 +205,7 @@ jobs:
           ref: ${{ env.TAG_NAME }}
       - name: Await release-notes published
         shell: bash
+        timeout-minutes: 120
         run: |
           until .ci/release/wait_release_notes_published.sh ${{ inputs.version }}
           do

From b32f4b03a55fa0e18866ec2cb531f9c260a88fd0 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 17:24:05 +0200
Subject: [PATCH 18/29] Remove dry_run input

---
 .ci/release.sh                | 7 +------
 .github/workflows/release.yml | 6 ------
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/.ci/release.sh b/.ci/release.sh
index dd5a516911..e00d7eb05e 100755
--- a/.ci/release.sh
+++ b/.ci/release.sh
@@ -25,9 +25,4 @@ java -version
 
 set +x
 echo "--- Deploy the release :package:"
-if [[ "$dry_run" == "true" ]] ; then
-  # Build artifacts only
-  ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true | tee release.txt
-else
-  ./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt
-fi
+./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f342146e30..431b165d47 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,10 +19,6 @@ on:
         type: boolean
         required: true
         default: true
-      dry_run:
-        description: If set, run a dry-run release
-        default: false
-        type: boolean
 
 env:
   JAVA_VERSION: 17
@@ -78,8 +74,6 @@ jobs:
           printBuildLogs: false
           artifactName: lambda-zip
           artifactPath: "elastic-apm-agent/target/elastic-apm-java-aws-lambda-layer-*.zip"
-          buildEnvVars: |
-            dry_run=${{ inputs.dry_run || 'false' }}
 
 
   await_artifact_on_maven_central:

From 9798e9c5b08f6f3ecef0ce4e73c415644cf9b7a6 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 17:27:44 +0200
Subject: [PATCH 19/29] Add notification

---
 .github/workflows/release.yml | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 431b165d47..d886e5e37c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -122,7 +122,7 @@ jobs:
       - run: git push origin ${{ inputs.branch }}
 
 
-  build_docker_images:
+  build_and_push_docker_images:
     name: "Build and push docker images"
     runs-on: ubuntu-latest
     needs:
@@ -217,3 +217,28 @@ jobs:
             --title="Release ${{ inputs.version }}" \
             --notes="[Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }})
             ${{ needs.publish_aws_lambda.outputs.arn_content }}"
+          
+
+  notify:
+    if: always()
+    needs:
+      - prepare_release
+      - maven_central_deploy
+      - await_artifact_on_maven_central
+      - update_major_branch
+      - update_cloudfoundry
+      - build_and_push_docker_images
+      - publish_aws_lambda
+    runs-on: ubuntu-latest
+    steps:
+      - id: check
+        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        with:
+          needs: ${{ toJSON(needs) }}
+      - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
+        with:
+          status: ${{ steps.check.outputs.status }}
+          vaultUrl: ${{ secrets.VAULT_ADDR }}
+          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
+          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
+          slackChannel: "#apm-agent-java"

From c8b1979baf1792e79789ede34bfea9b4425c043d Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 18:00:15 +0200
Subject: [PATCH 20/29] Only allow single workflow at a time

---
 .github/workflows/release.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d886e5e37c..d4fbfba5f2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -28,6 +28,9 @@ env:
 permissions:
   contents: write
 
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   prepare_release:
     permissions:

From 6c4c3a23b94961218fd9cbabbb3351fee4a62355 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 5 Jun 2023 18:01:03 +0200
Subject: [PATCH 21/29] complete needs list

---
 .github/workflows/release.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d4fbfba5f2..2440ebe36d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -232,6 +232,7 @@ jobs:
       - update_cloudfoundry
       - build_and_push_docker_images
       - publish_aws_lambda
+      - create_github_release
     runs-on: ubuntu-latest
     steps:
       - id: check

From 73d72724dd8912ca6fcd0c0e0cd0247f55d17e28 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Tue, 6 Jun 2023 10:49:53 +0200
Subject: [PATCH 22/29] Error if no artifacts our found from buildkite

---
 .github/workflows/release.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2440ebe36d..b98f4ce23c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -77,6 +77,7 @@ jobs:
           printBuildLogs: false
           artifactName: lambda-zip
           artifactPath: "elastic-apm-agent/target/elastic-apm-java-aws-lambda-layer-*.zip"
+          artifactIfNoFilesFound: error
 
 
   await_artifact_on_maven_central:

From e6cafa0754f05200bf59c219c9bc0a5c4e15be68 Mon Sep 17 00:00:00 2001
From: Jan Calanog <nejcalanog@gmail.com>
Date: Tue, 6 Jun 2023 13:02:08 +0200
Subject: [PATCH 23/29] Use correct repo

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b98f4ce23c..bf0c49000c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -66,7 +66,7 @@ jobs:
     steps:
       - id: buildkite
         name: Run Deploy
-        uses: reakaleek/apm-pipeline-library/.github/actions/buildkite@feature/buildkite-download-artifacts
+        uses: elastic/apm-pipeline-library/.github/actions/buildkite@current
         with:
           vaultUrl: ${{ secrets.VAULT_ADDR }}
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}

From 9e66e33c2c357009a0ea1a7daac5e8fe780cc36f Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 7 Jun 2023 11:20:46 +0200
Subject: [PATCH 24/29] Set permissions on individual jobs

---
 .github/workflows/release.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bf0c49000c..505fa4ebb3 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,7 +26,7 @@ env:
   TAG_NAME: v${{ inputs.version }}
 
 permissions:
-  contents: write
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}
@@ -102,6 +102,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - await_artifact_on_maven_central
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v3
         with:
@@ -115,6 +117,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - await_artifact_on_maven_central
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v3
         with:
@@ -197,6 +201,8 @@ jobs:
       - publish_aws_lambda
       - update_major_branch
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v3
         with:

From 90e40ef52562140d66b861403a72fddc480e300b Mon Sep 17 00:00:00 2001
From: Jan Calanog <nejcalanog@gmail.com>
Date: Wed, 7 Jun 2023 11:31:48 +0200
Subject: [PATCH 25/29] Update .github/workflows/release.yml

Co-authored-by: Adrien Mannocci <adrien.mannocci@gmail.com>
---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 505fa4ebb3..ce19419b82 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,7 +22,7 @@ on:
 
 env:
   JAVA_VERSION: 17
-  JAVA_DIST: adopt
+  JAVA_DIST: temurin
   TAG_NAME: v${{ inputs.version }}
 
 permissions:

From b5fe13f97fe01236306ecbb9c348caa7e5803630 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Wed, 7 Jun 2023 15:30:29 +0200
Subject: [PATCH 26/29] Remove comment from copy-paste

---
 .ci/release.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.ci/release.sh b/.ci/release.sh
index e00d7eb05e..13e30562a1 100755
--- a/.ci/release.sh
+++ b/.ci/release.sh
@@ -1,6 +1,4 @@
 #!/usr/bin/env bash
-##  This script runs the release given the different environment variables
-##    dry_run
 ##
 ##  It relies on the .buildkite/hooks/pre-command so the Vault and other tooling
 ##  are prepared automatically by buildkite.

From 4fbf5044d8379ce5727918e5a23f5823c79f8774 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 12 Jun 2023 12:09:52 +0200
Subject: [PATCH 27/29] Fix label

---
 .buildkite/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/release.yml b/.buildkite/release.yml
index 7f51edc121..1b6a41fae7 100644
--- a/.buildkite/release.yml
+++ b/.buildkite/release.yml
@@ -2,7 +2,7 @@ agents:
   provider: "gcp"
 
 steps:
-  - label: "Run the snapshot"
+  - label: "Run the release"
     key: "release"
     commands: .ci/release.sh
     artifact_paths:

From a967347e75c51abedc60beb128a56dc23fb6401d Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 12 Jun 2023 12:51:35 +0200
Subject: [PATCH 28/29] Add default aws region env

---
 .github/workflows/release.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ce19419b82..971026db7f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -162,6 +162,10 @@ jobs:
       - await_artifact_on_maven_central
     outputs:
       arn_content: ${{ steps.arn_output.outputs.arn_content }}
+    env:
+      # Random region. This needs to be set in GH Actions or the usage of aws-cli will fail.
+      # The default region does not matter, since we are publishing in all regions.
+      AWS_DEFAULT_REGION: eu-west-1
     steps:
       - uses: actions/checkout@v3
         with:

From 38fe1c41064edc3d620586ff249335d1c5cbc881 Mon Sep 17 00:00:00 2001
From: Jan Calanog <jan.calanog@elastic.co>
Date: Mon, 12 Jun 2023 13:45:03 +0200
Subject: [PATCH 29/29] Move benchmarks script to a more appropriate folder

---
 {scripts/docker-release => .ci}/run-benchmarks.sh | 0
 Jenkinsfile                                       | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename {scripts/docker-release => .ci}/run-benchmarks.sh (100%)

diff --git a/scripts/docker-release/run-benchmarks.sh b/.ci/run-benchmarks.sh
similarity index 100%
rename from scripts/docker-release/run-benchmarks.sh
rename to .ci/run-benchmarks.sh
diff --git a/Jenkinsfile b/Jenkinsfile
index 313b73bea9..9f804d6a9f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -131,7 +131,7 @@ pipeline {
               unstashV2(name: 'build', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
               dir("${BASE_DIR}"){
                 withOtelEnv() {
-                  sh './scripts/docker-release/run-benchmarks.sh'
+                  sh './.ci/run-benchmarks.sh'
                 }
               }
             }