diff --git a/.buildkite/release.yml b/.buildkite/release.yml new file mode 100644 index 0000000000..1b6a41fae7 --- /dev/null +++ b/.buildkite/release.yml @@ -0,0 +1,14 @@ +agents: + provider: "gcp" + +steps: + - label: "Run the release" + key: "release" + commands: .ci/release.sh + artifact_paths: + - "release.txt" + - "**/target/*" + +notify: + - slack: "#apm-agent-java" + if: 'build.state != "passed"' diff --git a/.ci/ReleaseChangelog.java b/.ci/ReleaseChangelog.java new file mode 100644 index 0000000000..8d6092e0e1 --- /dev/null +++ b/.ci/ReleaseChangelog.java @@ -0,0 +1,126 @@ +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.OptionalInt; +import java.util.function.Predicate; + +public class ReleaseChangelog { + + public static void main(String[] args) throws IOException { + if (args.length != 2) { + System.out.println("Expected exactly two arguments: "); + System.exit(-1); + } + Path fileName = Paths.get(args[0]); + String version = args[1].trim(); + if (!version.matches("\\d+\\.\\d+\\.\\d+")) { + System.out.println("Version must be in the format x.x.x but was not: " + version); + System.exit(-1); + } + Lines f = new Lines(Files.readAllLines(fileName, StandardCharsets.UTF_8)); + int unreleasedStart = f.findLine(str -> str.trim().equals("=== Unreleased"), 0).orElseThrow() + 1; + int unreleasedEnd = f.findLine(str -> str.startsWith("[[release-notes-"), unreleasedStart).orElseThrow(); + + Lines changes = f.cut(unreleasedStart, unreleasedEnd); + f.insert(new Lines(List.of("")), unreleasedStart); //add a blank line below unreleased heading + changes.trim(); + + // a few sanity checks + if (changes.lineCount() == 0) { + System.out.println("Unreleased changes are empty, there must be something wrong!"); + System.exit(-1); + } + OptionalInt wrongIndentedHeading = changes.findLine(str -> str.matches("^==?=?=?[^=].*"), 0); + if (wrongIndentedHeading.isPresent()) { + System.out.println("Found heading which is too high level (must be at least =====) within changes: " + + changes.getLine(wrongIndentedHeading.getAsInt())); + System.exit(-1); + } + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy/MM/dd") + .withZone(ZoneOffset.UTC); + + changes.insert(new Lines(List.of( + "", + String.format("[[release-notes-%s]]", version), + String.format("==== %s - %s", version, formatter.format(Instant.now())), + "" + )), 0); + + int majorVersion = Integer.parseInt(version.split("\\.")[0]); + String majorHeading = String.format("=== Java Agent version %d.x", majorVersion); + OptionalInt sectionStart = f.findLine(str -> str.trim().equals(majorHeading), 0); + if (sectionStart.isEmpty()) { + System.out.println("Could not find heading for major version: " + majorHeading); + System.out.println("Is this a new major version? If yes, please add an empty section for it manually to the Changelog"); + System.exit(-1); + } + + f.insert(changes, sectionStart.getAsInt() + 1); + + Files.writeString(fileName, f.toString(), StandardCharsets.UTF_8); + } + + static class Lines { + + private final List lines; + + public Lines(List lines) { + this.lines = new ArrayList<>(lines); + } + + int lineCount() { + return lines.size(); + } + + OptionalInt findLine(Predicate condition, int startAt) { + for (int i = startAt; i < lines.size(); i++) { + if (condition.test(lines.get(i))) { + return OptionalInt.of(i); + } + } + return OptionalInt.empty(); + } + + Lines cut(int startInclusive, int endExclusive) { + List cutLines = new ArrayList<>(); + for (int i = startInclusive; i < endExclusive; i++) { + cutLines.add(lines.remove(startInclusive)); + } + return new Lines(cutLines); + } + + void insert(Lines other, int insertAt) { + this.lines.addAll(insertAt, other.lines); + } + + /** + * Trims lines consisting of only blanks at the top and bottom + */ + void trim() { + while (!lines.isEmpty() && lines.get(0).matches("\\s*")) { + lines.remove(0); + } + while (!lines.isEmpty() && lines.get(lines.size() - 1).matches("\\s*")) { + lines.remove(lines.size() - 1); + } + } + + @Override + public String toString() { + return String.join("\n", lines); + } + + public String getLine(int number) { + return lines.get(number); + } + } + +} diff --git a/.ci/release.sh b/.ci/release.sh new file mode 100755 index 0000000000..13e30562a1 --- /dev/null +++ b/.ci/release.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +## +## It relies on the .buildkite/hooks/pre-command so the Vault and other tooling +## are prepared automatically by buildkite. +## + +set -eo pipefail + +# Make sure we delete this folder before leaving even in case of failure +clean_up () { + ARG=$? + export VAULT_TOKEN=$PREVIOUS_VAULT_TOKEN + echo "--- Deleting tmp workspace" + rm -rf $TMP_WORKSPACE + exit $ARG +} +trap clean_up EXIT + +echo "--- Debug JDK installation :coffee:" +echo $JAVA_HOME +echo $PATH +java -version + +set +x +echo "--- Deploy the release :package:" +./mvnw -V -s .ci/settings.xml -Pgpg clean deploy -DskipTests --batch-mode | tee release.txt diff --git a/.ci/release/Jenkinsfile b/.ci/release/Jenkinsfile index c2883860a4..99267a1b9f 100644 --- a/.ci/release/Jenkinsfile +++ b/.ci/release/Jenkinsfile @@ -97,7 +97,7 @@ pipeline { // If this fails, an exception should be thrown and execution will halt dir("${BASE_DIR}"){ script { - def r = sh(label: "Check Maven status", script: "./scripts/jenkins/check_maven.sh -u https://status.maven.org/api/v2/summary.json --component OSSRH", returnStatus: true) + def r = sh(label: "Check Maven status", script: "./scripts/docker-release/check_maven.sh -u https://status.maven.org/api/v2/summary.json --component OSSRH", returnStatus: true) if (r == 1) { error("Failing release build because Maven is the OSSRH component is not fully operational. See https://status.maven.org/ for more details.") } @@ -264,10 +264,10 @@ pipeline { dir("${BASE_DIR}"){ // fetch agent artifact from remote repository withEnv(["SONATYPE_FALLBACK=1"]) { - sh(label: "Build Docker image", script: "./scripts/jenkins/build_docker.sh") + sh(label: "Build Docker image", script: "./scripts/docker-release/build_docker.sh") // Get Docker registry credentials dockerLogin(secret: "${ELASTIC_DOCKER_SECRET}", registry: 'docker.elastic.co', role_id: 'apm-vault-role-id', secret_id: 'apm-vault-secret-id') - sh(label: "Push Docker image", script: "./scripts/jenkins/push_docker.sh") + sh(label: "Push Docker image", script: "./scripts/docker-release/push_docker.sh") } } } diff --git a/scripts/jenkins/run-benchmarks.sh b/.ci/run-benchmarks.sh similarity index 100% rename from scripts/jenkins/run-benchmarks.sh rename to .ci/run-benchmarks.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..971026db7f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,259 @@ +--- +# Releases the agent +name: release + +on: + workflow_dispatch: + inputs: + branch: + description: 'The branch to release' + required: true + default: 'main' + version: + description: 'The version to release (e.g. 1.2.3). This workflow will automatically perform the required version bumps' + required: true + update_changelog: + description: | + If enabled, everything in the changelog from the "Unreleased" section will be automatically moved to a new section for the new release. + If disabled, the changelog needs to be prepared for the release manually before triggering this workflow. + type: boolean + required: true + default: true + +env: + JAVA_VERSION: 17 + JAVA_DIST: temurin + TAG_NAME: v${{ inputs.version }} + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + +jobs: + prepare_release: + permissions: + contents: write + name: "Changelog and Version Bump" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v3 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_DIST }} + cache: 'maven' + - name: Prepare changelog for release + if: ${{ inputs.update_changelog }} + run: | + java .ci/ReleaseChangelog.java CHANGELOG.asciidoc ${{ inputs.version }} + git commit -m "Prepare changelog for release ${{ inputs.version }}" CHANGELOG.asciidoc + - name: Bump version and add git tag + run: ./mvnw release:prepare -B -DpushChanges=false "-Darguments=-DskipTests -Dmaven.javadoc.skip=true" -DreleaseVersion=${{ inputs.version }} + - run: git push --atomic origin ${{ inputs.branch }} ${{ env.TAG_NAME }} + + + maven_central_deploy: + name: "Deploy to Maven Central (Buildkite)" + runs-on: ubuntu-latest + needs: + - prepare_release + steps: + - id: buildkite + name: Run Deploy + uses: elastic/apm-pipeline-library/.github/actions/buildkite@current + with: + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} + pipeline: apm-agent-java-release + pipelineVersion: ${{ env.TAG_NAME }} + waitFor: true + printBuildLogs: false + artifactName: lambda-zip + artifactPath: "elastic-apm-agent/target/elastic-apm-java-aws-lambda-layer-*.zip" + artifactIfNoFilesFound: error + + + await_artifact_on_maven_central: + name: "Wait for artifacts to be available on maven central" + runs-on: ubuntu-latest + needs: + - maven_central_deploy + steps: + - uses: actions/checkout@v3 + - name: Await artifacts published in maven central + shell: bash + timeout-minutes: 120 + run: | + until .ci/release/wait_maven_artifact_published.sh ${{ inputs.version }} + do + echo "Artifacts not found on maven central. Sleeping 30 seconds, retrying afterwards" + sleep 30s + done + + update_major_branch: + name: "Update Major Branch" + runs-on: ubuntu-latest + needs: + - await_artifact_on_maven_central + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.TAG_NAME }} + - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current + - run: .ci/release/update_major_branch.sh ${{ inputs.version }} + - run: git push -f origin "$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')" + + update_cloudfoundry: + name: "Update Cloudfoundry" + runs-on: ubuntu-latest + needs: + - await_artifact_on_maven_central + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current + - name: "Update Cloudfoundry index.yml file" + shell: bash + run: .ci/release/update_cloudfoundry.sh ${{ inputs.version }} + - run: git push origin ${{ inputs.branch }} + + + build_and_push_docker_images: + name: "Build and push docker images" + runs-on: ubuntu-latest + needs: + - await_artifact_on_maven_central + env: + SONATYPE_FALLBACK: 1 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.TAG_NAME }} + fetch-depth: 0 # Load entire history as it is required for the push-script + - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current + with: + registry: docker.elastic.co + secret: secret/apm-team/ci/docker-registry/prod + url: ${{ secrets.VAULT_ADDR }} + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + - name: "Build docker image" + shell: bash + run: | + ./scripts/docker-release/build_docker.sh + ./scripts/docker-release/push_docker.sh + + publish_aws_lambda: + name: "Publish AWS Lambda" + runs-on: ubuntu-latest + needs: + - await_artifact_on_maven_central + outputs: + arn_content: ${{ steps.arn_output.outputs.arn_content }} + env: + # Random region. This needs to be set in GH Actions or the usage of aws-cli will fail. + # The default region does not matter, since we are publishing in all regions. + AWS_DEFAULT_REGION: eu-west-1 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.TAG_NAME }} + - uses: actions/download-artifact@v3 + with: + name: lambda-zip + path: elastic-apm-agent/target + - uses: hashicorp/vault-action@v2.4.2 + with: + url: ${{ secrets.VAULT_ADDR }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + secrets: | + secret/observability-team/ci/service-account/apm-aws-lambda access_key_id | AWS_ACCESS_KEY_ID ; + secret/observability-team/ci/service-account/apm-aws-lambda secret_access_key | AWS_SECRET_ACCESS_KEY + - name: Publish + run: make -C .ci publish-in-all-aws-regions + - name: Create ARN file + run: make -C .ci create-arn-file + - uses: actions/upload-artifact@v3 + with: + name: arn-file + path: .ci/arn-file.md + - name: Add ARN file to output + id: arn_output + run: | + echo 'arn_content<> $GITHUB_OUTPUT + cat .ci/arn-file.md >> $GITHUB_OUTPUT + echo 'ARN_CONTENT_EOF' >> $GITHUB_OUTPUT + + + create_github_release: + name: "Create GitHub Release" + needs: + - publish_aws_lambda + - update_major_branch + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.TAG_NAME }} + - name: Await release-notes published + shell: bash + timeout-minutes: 120 + run: | + until .ci/release/wait_release_notes_published.sh ${{ inputs.version }} + do + echo "Release notes not published yet. Sleeping 30 seconds, retrying afterwards" + sleep 30s + done + - name: Compute major.x branch + id: get_dotx_branch + run: echo "dotx_branch=$(echo '${{ inputs.version }}' | sed -E 's/\..+/.x/')" >> $GITHUB_OUTPUT + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create ${{ env.TAG_NAME }} \ + --title="Release ${{ inputs.version }}" \ + --notes="[Release Notes for ${{ inputs.version }}](https://www.elastic.co/guide/en/apm/agent/java/current/release-notes-${{ steps.get_dotx_branch.outputs.dotx_branch }}.html#release-notes-${{ inputs.version }}) + ${{ needs.publish_aws_lambda.outputs.arn_content }}" + + + notify: + if: always() + needs: + - prepare_release + - maven_central_deploy + - await_artifact_on_maven_central + - update_major_branch + - update_cloudfoundry + - build_and_push_docker_images + - publish_aws_lambda + - create_github_release + runs-on: ubuntu-latest + steps: + - id: check + uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current + with: + needs: ${{ toJSON(needs) }} + - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + with: + status: ${{ steps.check.outputs.status }} + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} + slackChannel: "#apm-agent-java" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 468d2ce4a0..8707aee2de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -315,7 +315,7 @@ For illustration purpose, `1.2.3` will be the target release version, and the gi 1. Download `elastic-apm-java-aws-lambda-layer-.zip` from the CI release job artifacts and upload it to the release draft 1. Wait for released package to be available in [maven central](https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/) 1. Build and push a Docker image using the instructions below - Use `SONATYPE_FALLBACK=1 scripts/jenkins/build_docker.sh` to build image with released artifact. + Use `SONATYPE_FALLBACK=1 scripts/docker-release/build_docker.sh` to build image with released artifact. Requires credentials, thus need to delegate this manual step to someone that has them. 1. Update [`cloudfoundry/index.yml`](cloudfoundry/index.yml) on `main`. 1. Publish release on Github. This will notify users watching repository. @@ -336,7 +336,7 @@ docker pull docker.elastic.co/observability/apm-agent-java:1.12.0 #### Creating images for a Release Images are normally created and published as a part of the release process. -Scripts which manage the Docker release process are located in [`scripts/jenkins/`](scripts/jenkins/). +Scripts which manage the Docker release process are located in [`scripts/docker-release/`](scripts/docker-release/). ##### Building images locally @@ -347,13 +347,13 @@ Building images on a workstation requires the following: * A local checkout of the `apm-agent-java` repo * The `git` command-line tool -Local image building is handled via the [`build_docker.sh script`](scripts/jenkins/build_docker.sh). +Local image building is handled via the [`build_docker.sh script`](scripts/docker-release/build_docker.sh). If you wish to use a locally built artifact in the built image, execute [`./mvnw package`](mvnw)` and ensure that artifacts are present in `elastic-apm-agent/target/*.jar`. To create a Docker image from artifacts generated by [`./mvnw package`](mvnw), -run [`scripts/jenkins/build_docker.sh`](scripts/jenkins/build_docker.sh). +run [`scripts/docker-release/build_docker.sh`](scripts/docker-release/build_docker.sh). Alternatively, it is also possible to use the most recent artifact from the [Sonatype repository](https://oss.sonatype.org/#nexus-search;gav~co.elastic.apm~apm-agent-java~~~). @@ -361,16 +361,16 @@ repository](https://oss.sonatype.org/#nexus-search;gav~co.elastic.apm~apm-agent- To do so, first clean any artifacts with [`./mvnw clean`](mvnw) and then run the Docker build script with the `SONATYPE_FALLBACK` environment variable present. For example, -`SONATYPE_FALLBACK=1 scripts/jenkins/build_docker.sh` +`SONATYPE_FALLBACK=1 scripts/docker-release/build_docker.sh` -After running the [`build_docker.sh`](scripts/jenkins/build_docker.sh) script, images can be seen by executing +After running the [`build_docker.sh`](scripts/docker-release/build_docker.sh) script, images can be seen by executing `docker images|egrep docker.elastic.co/observability/apm-agent-java` which should produce output similar to the following: `docker.elastic.co/observability/apm-agent-java 1.12.0 1f45b5858d81 26 hours ago 10.6MB` No output from the above command indicates that the image did not build correctly -and that the output of the [`build_docker.sh`](scripts/jenkins/build_docker.sh) +and that the output of the [`build_docker.sh`](scripts/docker-release/build_docker.sh) script should be examined to determine the cause. @@ -381,5 +381,5 @@ _Notice:_ You must have access to release secrets in order to push images. Prior to pushing images, you must login to the Elastic Docker repo using the correct credentials using the [`docker login`](https://docs.docker.com/engine/reference/commandline/login/) command. -To push an image, run the [`scripts/jenkins/push_docker.sh`](scripts/jenkins/push_docker.sh) +To push an image, run the [`scripts/docker-release/push_docker.sh`](scripts/docker-release/push_docker.sh) script. An image will be pushed. diff --git a/Jenkinsfile b/Jenkinsfile index 8d494125ef..9f804d6a9f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -131,7 +131,7 @@ pipeline { unstashV2(name: 'build', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ withOtelEnv() { - sh './scripts/jenkins/run-benchmarks.sh' + sh './.ci/run-benchmarks.sh' } } } diff --git a/scripts/jenkins/build_docker.sh b/scripts/docker-release/build_docker.sh similarity index 100% rename from scripts/jenkins/build_docker.sh rename to scripts/docker-release/build_docker.sh diff --git a/scripts/jenkins/check_maven.sh b/scripts/docker-release/check_maven.sh similarity index 100% rename from scripts/jenkins/check_maven.sh rename to scripts/docker-release/check_maven.sh diff --git a/scripts/jenkins/fetch_nexus_id.sh b/scripts/docker-release/fetch_nexus_id.sh similarity index 100% rename from scripts/jenkins/fetch_nexus_id.sh rename to scripts/docker-release/fetch_nexus_id.sh diff --git a/scripts/jenkins/push_docker.sh b/scripts/docker-release/push_docker.sh similarity index 89% rename from scripts/jenkins/push_docker.sh rename to scripts/docker-release/push_docker.sh index e6f26bd0ee..2953639ce9 100755 --- a/scripts/jenkins/push_docker.sh +++ b/scripts/docker-release/push_docker.sh @@ -48,7 +48,7 @@ if [ ${WORKERS+x} ] # We are on a CI worker then retry $RETRIES docker push $DOCKER_PUSH_IMAGE || echo "Push failed after $RETRIES retries" else # We are in a local (non-CI) environment - docker push $DOCKER_PUSH_IMAGE || echo "You may need to run 'docker login' first and then re-run this script" + docker push $DOCKER_PUSH_IMAGE || { echo "You may need to run 'docker login' first and then re-run this script"; exit 1; } fi readonly LATEST_TAG=$(git tag --list --sort=version:refname "v*" | grep -v RC | sed s/^v// | tail -n 1) @@ -62,6 +62,6 @@ then then retry $RETRIES docker push $DOCKER_PUSH_IMAGE_LATEST || echo "Push failed after $RETRIES retries" else # We are in a local (non-CI) environment - docker push $DOCKER_PUSH_IMAGE_LATEST || echo "You may need to run 'docker login' first and then re-run this script" + docker push $DOCKER_PUSH_IMAGE_LATEST || { echo "You may need to run 'docker login' first and then re-run this script"; exit 1; } fi fi