Arp demonstration

.buildkite/hooks/post-checkout

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""

.buildkite/hooks/pre-command

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+# Avoid any pager when running git commands
+git config --global core.pager 'cat'
+
+export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
+export BASE_DIR=$(pwd)
+export GO_VERSION=$(cat .go-version)
+
+export REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
+export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
+export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+
+export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+
+JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
+SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
+PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
+PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials
+
+EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
+EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
+
+if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
+    # first upload pipeline does not have the environment variables defined in the YAML
+    export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
+fi
+
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
+    if [ ${BUILDKITE_STEP_KEY} == "publish-packages" ]; then
+        export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
+        export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
+        export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+
+        # signing job
+        export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+
+        # publishing job
+        export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
+    fi
+fi
+
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations" ]; then
+    # FIXME: update condition depending on the pipeline steps triggered
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations" ]]; then
+        export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+        export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+        export JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
+        # Environment variables required by the service deployer
+        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+    fi
+fi
+
+if [ ${BUILDKITE_PIPELINE_SLUG} == "integrations-serverless" ]; then
+    if [[ ${BUILDKITE_STEP_KEY} == "test-integrations-serverless-project" ]]; then
+        export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+        export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+        export JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
+
+        # Environment variables required by the service deployer
+        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+
+        export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey ${EC_TOKEN_PATH})
+        export EC_HOST_SECRET=$(retry 5 vault kv get -field url ${EC_TOKEN_PATH})
+        export EC_REGION_SECRET=$(retry 5 vault read -field region_qa ${EC_DATA_PATH})
+    fi
+fi

.buildkite/hooks/pre-exit

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
+    # FIXME: update condition depending on the pipeline steps triggered
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations" ]]; then
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
+        if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            echo "--- Take down the Elastic stack"
+            ${ELASTIC_PACKAGE_BIN} stack down -v
+        fi
+    fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
+        if [ -f ${ELASTIC_PACKAGE_BIN} ]; then
+            echo "--- Take down the Elastic stack"
+            export EC_API_KEY=${EC_API_KEY_SECRET}
+            export EC_HOST=${EC_HOST_SECRET}
+
+            ${ELASTIC_PACKAGE_BIN} stack down -v
+
+            unset EC_API_KEY
+            unset EC_HOST
+        fi
+    fi
+fi
+
+unset_secrets
+cleanup

.buildkite/pipeline.schedule-daily.yml

@@ -0,0 +1,36 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: integrations-schedule-daily
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+
+# The pipeline is triggered by the scheduler every day
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations in serverless - project: Observability"
+    key: "trigger-integrations-serverless-obs"
+    trigger: "integrations-serverless"
+    build:
+      env:
+        SERVERLESS_PROJECT: observability
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
+  - label: "Check integrations in serverless - project: Security"
+    key: "trigger-integrations-serverless-security"
+    trigger: "integrations-serverless"
+    build:
+      env:
+        SERVERLESS_PROJECT: security
+    depends_on:
+      - step: "check"
+        allow_failure: false

.buildkite/pipeline.serverless.yml

@@ -0,0 +1,73 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.27.3'
+  YQ_VERSION: 'v4.35.2'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+
+steps:
+  - input: "Input values for the variables"
+    key: "input-variables"
+    fields:
+    - select: "SERVERLESS_PROJECT"
+      key: "SERVERLESS_PROJECT"
+      options:
+        - label: "observability"
+          value: "observability"
+        - label: "security"
+          value: "security"
+      default: "observability"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations in serverless"
+    key: "test-integrations-serverless-project"
+    command: ".buildkite/scripts/test_integrations_with_serverless.sh"
+    timeout_in_minutes: 120
+    env:
+      SERVERLESS: true
+      FORCE_CHECK_ALL: true
+      UPLOAD_SAFE_LOGS: 1
+    agents:
+      provider: "gcp"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      # - "build/elastic-stack-dump/*/logs/*.log"
+      # - "build/elastic-stack-dump/*/logs/fleet-server-internal/**/*"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker

.buildkite/pipeline.yml

@@ -1,5 +1,55 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.27.3'
+  YQ_VERSION: 'v4.35.2'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  # TODO: Pending to migrate stages from https://github.com/elastic/integrations/blob/993537b80456edf2035f2a4826031841116c2019/.ci/Jenkinsfile
+  # - label: "Publish packages - INCOMPLETE"
+  #   key: "publish-packages"
+  #   command: ".buildkite/scripts/publish_packages.sh" # TODO: missing signature and publishing loops
+  #   agents:
+  #     provider: "gcp"
+  #   depends_on:
+  #     - step: "check"
+  #       allow_failure: false
+
+  # - label: "Trigger integrations"
+  #   key: "trigger-integrations"
+  #   command: ".buildkite/scripts/trigger_integrations_in_parallel.sh" # TODO: missing script
+  #   depends_on:
+  #     - step: "publish-packages"
+  #       allow_failure: false
+
+  # - wait: ~
+  #   continue_on_failure: true
+
+  # - label: ":junit: Junit annotate"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: "build/test-results/*.xml"
+  #   agents:
+  #     provider: "gcp"  # junit plugin requires docker