From 173de5b7f87f226fed31ca799781555e1080a682 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 25 Jan 2024 10:30:48 +0100 Subject: [PATCH 1/3] updatecli: change location --- .ci/{ => updatecli}/updatecli.d/update-gherkin-specs.yml | 0 .ci/{ => updatecli}/updatecli.d/update-json-specs.yml | 0 .ci/{ => updatecli}/updatecli.d/update-specs.yml | 0 .github/workflows/updatecli.yml | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename .ci/{ => updatecli}/updatecli.d/update-gherkin-specs.yml (100%) rename .ci/{ => updatecli}/updatecli.d/update-json-specs.yml (100%) rename .ci/{ => updatecli}/updatecli.d/update-specs.yml (100%) diff --git a/.ci/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml similarity index 100% rename from .ci/updatecli.d/update-gherkin-specs.yml rename to .ci/updatecli/updatecli.d/update-gherkin-specs.yml diff --git a/.ci/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml similarity index 100% rename from .ci/updatecli.d/update-json-specs.yml rename to .ci/updatecli/updatecli.d/update-json-specs.yml diff --git a/.ci/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml similarity index 100% rename from .ci/updatecli.d/update-specs.yml rename to .ci/updatecli/updatecli.d/update-specs.yml diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 2101ec798..618c814ae 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -18,7 +18,7 @@ jobs: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: .ci/updatecli.d + pipeline: .ci/updatecli/updatecli.d - if: failure() uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current with: From 9ca66e3dcb3161cad51e3fc3d31b566341008609 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 25 Jan 2024 10:32:48 +0100 Subject: [PATCH 2/3] updatecli: use values --- .ci/updatecli/values.yml | 14 ++++++++++++++ .github/workflows/updatecli.yml | 1 + 2 files changed, 15 insertions(+) create mode 100644 .ci/updatecli/values.yml diff --git a/.ci/updatecli/values.yml b/.ci/updatecli/values.yml new file mode 100644 index 000000000..b0b58d73e --- /dev/null +++ b/.ci/updatecli/values.yml @@ -0,0 +1,14 @@ +github: + owner: "elastic" + repository: "apm-agent-python" + apm_repository: "apm" + apm_data_repository: "apm-data" + branch: "main" +specs: + apm_data_path: "input/elasticapm/docs/spec/v2" + apm_json_path: "tests/agents/json-specs" + apm_gherkin_path: "tests/agents/gherkin-specs" +apm_agent: + gherkin_specs_path: "tests/bdd/features" + json_specs_path: "tests/upstream/json-specs" + server_schema_specs_path: "tests/upstream/json-specs" \ No newline at end of file diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 618c814ae..22598200b 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -19,6 +19,7 @@ jobs: vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: .ci/updatecli/updatecli.d + values: .ci/updatecli/values.yml - if: failure() uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current with: From 8cc52e6f902dc03525dc5e440665cecfa97f82fe Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 25 Jan 2024 10:33:10 +0100 Subject: [PATCH 3/3] updatecli: use dynamic approach --- .../updatecli.d/update-gherkin-specs.yml | 117 ++++++---------- .../updatecli.d/update-json-specs.yml | 128 ++++++------------ .ci/updatecli/updatecli.d/update-specs.yml | 110 +++++++-------- 3 files changed, 133 insertions(+), 222 deletions(-) diff --git a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml index 8deb269fc..f12ece861 100644 --- a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml +++ b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml @@ -1,6 +1,5 @@ name: update-gherkin-specs pipelineid: update-gherkin-specs -title: synchronize gherkin specs scms: default: @@ -8,52 +7,55 @@ scms: spec: user: '{{ requiredEnv "GIT_USER" }}' email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' username: '{{ requiredEnv "GIT_USER" }}' - branch: main + branch: "{{ .github.branch }}" + apm: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" sources: sha: kind: file spec: - file: 'https://github.com/elastic/apm/commit/main.patch' + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' matchpattern: "^From\\s([0-9a-f]{40})\\s" transformers: - findsubmatch: pattern: "[0-9a-f]{40}" - - api_key.feature: - kind: file + pull_request: + kind: shell + dependson: + - sha spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/api_key.feature - azure_app_service_metadata.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/azure_app_service_metadata.feature - azure_functions_metadata.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/azure_functions_metadata.feature - otel_bridge.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/otel_bridge.feature - outcome.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/outcome.feature - user_agent.feature: - kind: file + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agents-gherkin-specs-tarball: + kind: shell + scmid: apm + dependson: + - sha spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/user_agent.feature + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_gherkin_path }}" actions: pr: kind: "github/pullrequest" scmid: default - title: '[Automation] Update Gherkin specs' spec: automerge: false draft: false @@ -62,56 +64,21 @@ actions: description: |- ### What APM agent Gherkin specs automatic sync + ### Why *Changeset* + * {{ source "pull_request" }} * https://github.com/elastic/apm/commit/{{ source "sha" }} + title: '[Automation] Update Gherkin specs' targets: - api_key.feature: - name: api_key.feature + agent-gherkin-specs: + name: APM agent gherkin specs {{ source "sha" }} scmid: default - sourceid: api_key.feature - kind: file - spec: - file: tests/bdd/features/api_key.feature - forcecreate: true - azure_app_service_metadata.feature: - name: azure_app_service_metadata.feature - scmid: default - sourceid: azure_app_service_metadata.feature - kind: file - spec: - file: tests/bdd/features/azure_app_service_metadata.feature - forcecreate: true - azure_functions_metadata.feature: - name: azure_functions_metadata.feature - scmid: default - sourceid: azure_functions_metadata.feature - kind: file - spec: - file: tests/bdd/features/azure_functions_metadata.feature - forcecreate: true - otel_bridge.feature: - name: otel_bridge.feature - scmid: default - sourceid: otel_bridge.feature - kind: file - spec: - file: tests/bdd/features/otel_bridge.feature - forcecreate: true - outcome.feature: - name: outcome.feature - scmid: default - sourceid: outcome.feature - kind: file - spec: - file: tests/bdd/features/outcome.feature - forcecreate: true - user_agent.feature: - name: user_agent.feature - scmid: default - sourceid: user_agent.feature - kind: file + disablesourceinput: true + kind: shell spec: - file: tests/bdd/features/user_agent.feature - forcecreate: true + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.gherkin_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml index 13d25c834..e05aaecdb 100644 --- a/.ci/updatecli/updatecli.d/update-json-specs.yml +++ b/.ci/updatecli/updatecli.d/update-json-specs.yml @@ -1,6 +1,5 @@ name: update-json-specs pipelineid: update-json-specs -title: synchronize json specs scms: default: @@ -8,56 +7,55 @@ scms: spec: user: '{{ requiredEnv "GIT_USER" }}' email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' username: '{{ requiredEnv "GIT_USER" }}' - branch: main + branch: "{{ .github.branch }}" + apm: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" sources: sha: kind: file spec: - file: 'https://github.com/elastic/apm/commit/main.patch' + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' matchpattern: "^From\\s([0-9a-f]{40})\\s" transformers: - findsubmatch: pattern: "[0-9a-f]{40}" - - container_metadata_discovery.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/container_metadata_discovery.json - service_resource_inference.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/service_resource_inference.json - span_types.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/span_types.json - sql_signature_examples.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/sql_signature_examples.json - sql_token_examples.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/sql_token_examples.json - w3c_distributed_tracing.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/w3c_distributed_tracing.json - wildcard_matcher_tests.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/wildcard_matcher_tests.json + pull_request: + kind: shell + dependson: + - sha + spec: + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agents-json-specs-tarball: + kind: shell + scmid: apm + dependson: + - sha + spec: + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_json_path }}" actions: pr: kind: "github/pullrequest" scmid: default - title: '[Automation] Update JSON specs' spec: automerge: false draft: false @@ -66,57 +64,21 @@ actions: description: |- ### What APM agent specs automatic sync + ### Why *Changeset* - * https://github.com/elastic/apm/commit/{{ source "sha" }} + * {{ source "pull_request" }} + * https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ source "sha" }} + title: '[Automation] Update JSON specs' targets: - container_metadata_discovery.json: - name: container_metadata_discovery.json - scmid: default - sourceid: container_metadata_discovery.json - kind: file - spec: - file: tests/upstream/json-specs/container_metadata_discovery.json - service_resource_inference.json: - name: service_resource_inference.json - scmid: default - sourceid: service_resource_inference.json - kind: file - spec: - file: tests/upstream/json-specs/service_resource_inference.json - span_types.json: - name: span_types.json - scmid: default - sourceid: span_types.json - kind: file - spec: - file: tests/upstream/json-specs/span_types.json - sql_signature_examples.json: - name: sql_signature_examples.json + agent-json-specs: + name: APM agent json specs {{ source "sha" }} scmid: default - sourceid: sql_signature_examples.json - kind: file - spec: - file: tests/upstream/json-specs/sql_signature_examples.json - sql_token_examples.json: - name: sql_token_examples.json - scmid: default - sourceid: sql_token_examples.json - kind: file - spec: - file: tests/upstream/json-specs/sql_token_examples.json - w3c_distributed_tracing.json: - name: w3c_distributed_tracing.json - scmid: default - sourceid: w3c_distributed_tracing.json - kind: file - spec: - file: tests/upstream/json-specs/w3c_distributed_tracing.json - wildcard_matcher_tests.json: - name: wildcard_matcher_tests.json - scmid: default - sourceid: wildcard_matcher_tests.json - kind: file + disablesourceinput: true + kind: shell spec: - file: tests/upstream/json-specs/wildcard_matcher_tests.json + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.json_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml index ab3bd34c7..554140da2 100644 --- a/.ci/updatecli/updatecli.d/update-specs.yml +++ b/.ci/updatecli/updatecli.d/update-specs.yml @@ -1,6 +1,5 @@ name: update-specs pipelineid: update-schema-specs -title: synchronize schema specs scms: default: @@ -8,47 +7,57 @@ scms: spec: user: '{{ requiredEnv "GIT_USER" }}' email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' username: '{{ requiredEnv "GIT_USER" }}' - branch: main + branch: "{{ .github.branch }}" + + apm-data: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_data_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" sources: sha: kind: file spec: - file: 'https://github.com/elastic/apm-data/commit/main.patch' + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ .github.branch }}.patch' matchpattern: "^From\\s([0-9a-f]{40})\\s" transformers: - findsubmatch: pattern: "[0-9a-f]{40}" - error.json: - kind: file + pull_request: + kind: shell + dependson: + - sha spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/error.json - metadata.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/metadata.json - metricset.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/metricset.json - span.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/span.json - transaction.json: - kind: file + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_data_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agent-specs-tarball: + kind: shell + scmid: apm-data + dependson: + - sha spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/transaction.json + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_data_path }}" actions: pr: kind: "github/pullrequest" scmid: default - title: '[Automation] Update JSON schema specs' + sourceid: sha spec: automerge: false draft: false @@ -56,49 +65,22 @@ actions: - "automation" description: |- ### What - APM agent json schema automatic sync + APM agent json server schema automatic sync + ### Why *Changeset* - * https://github.com/elastic/apm-data/commit/{{ source "sha" }} + * {{ source "pull_request" }} + * https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ source "sha" }} + title: '[Automation] Update JSON server schema specs' targets: - error.json: - name: error.json - scmid: default - sourceid: error.json - kind: file - spec: - file: tests/upstream/json-specs/error.json - forcecreate: true - metadata.json: - name: metadata.json - scmid: default - sourceid: metadata.json - kind: file - spec: - file: tests/upstream/json-specs/metadata.json - forcecreate: true - metricset.json: - name: metricset.json + agent-json-schema: + name: APM agent json server schema {{ source "sha" }} scmid: default - sourceid: metricset.json - kind: file - spec: - file: tests/upstream/json-specs/metricset.json - forcecreate: true - span.json: - name: span.json - scmid: default - sourceid: span.json - kind: file - spec: - file: tests/upstream/json-specs/span.json - forcecreate: true - transaction.json: - name: transaction.json - scmid: default - sourceid: transaction.json - kind: file + disablesourceinput: true + kind: shell spec: - file: tests/upstream/json-specs/transaction.json - forcecreate: true + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.server_schema_specs_path }}"