diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml index 46f4c9559a7e..4cdce4fb7e7e 100644 --- a/.github/workflows/e2e.log.yaml +++ b/.github/workflows/e2e.log.yaml @@ -39,25 +39,37 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['h2', 'mysql', 'es6', 'es7', 'es7.14', 'influxdb'] - env: - SW_STORAGE: ${{ matrix.storage }} - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Set Skip Env Var - uses: ./.github/actions/skip - - name: Run E2E Test - if: env.SKIP_CI != 'true' - uses: ./.github/actions/e2e-test - with: - test_class: org.apache.skywalking.e2e.log.LogE2E + config-file: + - log/h2/e2e.yaml + - log/mysql/e2e.yaml + - log/influxdb/e2e.yaml + - log/postgres/e2e.yaml + include: + - es-version: 6.3.2 + config-file: log/es/e2e.yaml + - es-version: 7.0.0 + config-file: log/es/e2e.yaml + - es-version: 7.8.0 + config-file: log/es/e2e.yaml + - es-version: 7.10.1 + config-file: log/es/e2e.yaml + - es-version: 7.14.0 + config-file: log/es/e2e.yaml + - es-version: 7.15.0 + config-file: log/es/e2e.yaml - LogFluentBit: - if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule') - name: Log (FluentBit) - runs-on: ubuntu-latest + - es-version: 6.3.2 + config-file: log/fluent-bit/e2e.yaml + - es-version: 7.0.0 + config-file: log/fluent-bit/e2e.yaml + - es-version: 7.8.0 + config-file: log/fluent-bit/e2e.yaml + - es-version: 7.10.1 + config-file: log/fluent-bit/e2e.yaml + - es-version: 7.14.0 + config-file: log/fluent-bit/e2e.yaml + - es-version: 7.15.0 + config-file: log/fluent-bit/e2e.yaml steps: - uses: actions/checkout@v2 with: @@ -66,6 +78,8 @@ jobs: uses: ./.github/actions/skip - name: Run E2E Test if: env.SKIP_CI != 'true' - uses: ./.github/actions/e2e-test + uses: ./.github/actions/infra-e2e-test + env: + ES_VERSION: ${{ matrix.es-version }} with: - test_class: org.apache.skywalking.e2e.log.LogFluentBitE2E + config-file: ${{ matrix.config-file }} diff --git a/CHANGES.md b/CHANGES.md index 8cb1e16eb438..acae23e49422 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Release Notes. 8.9.0 ------------------ +#### Project + +* replace log e2e cases to e2e-v2. #### OAP Server diff --git a/test/e2e-v2/cases/log/es/docker-compose.yml b/test/e2e-v2/cases/log/es/docker-compose.yml new file mode 100644 index 000000000000..46edc680882c --- /dev/null +++ b/test/e2e-v2/cases/log/es/docker-compose.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:${ES_VERSION} + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + SW_STORAGE_ES_CLUSTER_NODES: es:9200 + ports: + - 12800 + depends_on: + es: + condition: service_healthy + networks: + - e2e + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + networks: + - e2e +networks: + e2e: diff --git a/test/e2e-v2/cases/log/es/e2e.yaml b/test/e2e-v2/cases/log/es/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/es/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file diff --git a/test/e2e-v2/cases/log/expected/logs.yml b/test/e2e-v2/cases/log/expected/logs.yml new file mode 100644 index 000000000000..594fdeedf63d --- /dev/null +++ b/test/e2e-v2/cases/log/expected/logs.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: +{{- contains .logs }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: | + {{ notEmpty .content }} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: {{ notEmpty .value }} + - key: thread + value: {{ notEmpty .value }} + {{- end }} +{{- end }} +total: {{ gt .total 0 }} \ No newline at end of file diff --git a/test/e2e-v2/cases/log/expected/service-endpoint.yml b/test/e2e-v2/cases/log/expected/service-endpoint.yml new file mode 100644 index 000000000000..dd58100f94f4 --- /dev/null +++ b/test/e2e-v2/cases/log/expected/service-endpoint.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains .}} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users +{{- end}} \ No newline at end of file diff --git a/test/e2e-v2/cases/log/expected/service-instance.yml b/test/e2e-v2/cases/log/expected/service-instance.yml new file mode 100644 index 000000000000..652b0137133c --- /dev/null +++ b/test/e2e-v2/cases/log/expected/service-instance.yml @@ -0,0 +1,40 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains .}} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + name: {{ notEmpty .name }} + attributes: + {{- contains .attributes }} + - name: OS Name + value: Linux + - name: hostname + value: {{ notEmpty .value }} + - name: Process No. + value: "1" + - name: Start Time + value: {{ notEmpty .value }} + - name: JVM Arguments + value: '{{ notEmpty .value }}' + - name: Jar Dependencies + value: '{{ notEmpty .value }}' + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: JAVA + instanceuuid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} +{{- end}} \ No newline at end of file diff --git a/test/e2e-v2/cases/log/expected/service.yml b/test/e2e-v2/cases/log/expected/service.yml new file mode 100644 index 000000000000..25f310ffe0ec --- /dev/null +++ b/test/e2e-v2/cases/log/expected/service.yml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-provider" }}.1 + name: e2e-service-provider + group: "" +{{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml new file mode 100644 index 000000000000..2a3e4a2a0088 --- /dev/null +++ b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:${ES_VERSION} + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + SW_STORAGE_ES_CLUSTER_NODES: es:9200 + depends_on: + es: + condition: service_healthy + ports: + - 12800 + networks: + - e2e + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + volumes: + - /tmp/skywalking-logs/:/tmp/skywalking-logs/ + ports: + - 9090 + networks: + - e2e + depends_on: + oap: + condition: service_healthy + + fluentbit: + image: fluent/fluent-bit:1.7 + volumes: + - /tmp/skywalking-logs:/tmp/skywalking-logs + - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + - ./fluent-bit-parser.conf:/fluent-bit/etc/fluent-bit-parser.conf + - ./fluent-bit-script.lua:/fluent-bit/etc/fluent-bit-script.lua + networks: + - e2e + depends_on: + oap: + condition: service_healthy + provider: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/fluent-bit/e2e.yaml b/test/e2e-v2/cases/log/fluent-bit/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/fluent-bit/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file diff --git a/test/e2e-v2/cases/log/fluent-bit/fluent-bit-parser.conf b/test/e2e-v2/cases/log/fluent-bit/fluent-bit-parser.conf new file mode 100644 index 000000000000..be298d631429 --- /dev/null +++ b/test/e2e-v2/cases/log/fluent-bit/fluent-bit-parser.conf @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[PARSER] + Name my-log-format + Format regex + Regex ^\[SW_CTX: ?\[(?[^,]+),(?[^,]+),(?[^,]+),(?[^,]+),(?[^\]]+)\]\] \[(?.+?)\] (?[^\]]*) \[(?[^ ]*)\] (?[^\]]*):(?[^\]]*) - (?[^\]]*)$ + Time_Key time + Time_Format %d/%b/%Y:%H:%M:%S %z diff --git a/test/e2e-v2/cases/log/fluent-bit/fluent-bit-script.lua b/test/e2e-v2/cases/log/fluent-bit/fluent-bit-script.lua new file mode 100644 index 000000000000..274d62e467eb --- /dev/null +++ b/test/e2e-v2/cases/log/fluent-bit/fluent-bit-script.lua @@ -0,0 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +function rewrite_body(tag, timestamp, record) + record["body"] = {text={text="fluentbit " .. record["body"]}} + record["tags"] = {data={{key="level", value="INFO"}}} + record["traceContext"] = {traceId=record["traceId"],traceSegmentId=record["traceSegmentId"],spanId=record["spanId"]} + return 1, timestamp, record +end diff --git a/test/e2e-v2/cases/log/fluent-bit/fluent-bit.conf b/test/e2e-v2/cases/log/fluent-bit/fluent-bit.conf new file mode 100644 index 000000000000..28d9554eef9b --- /dev/null +++ b/test/e2e-v2/cases/log/fluent-bit/fluent-bit.conf @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[SERVICE] + Flush 5 + Daemon Off + Log_Level warn + Parsers_File fluent-bit-parser.conf +[INPUT] + Name tail + Path /tmp/skywalking-logs/*/e2e-service-provider.log + Parser my-log-format +[FILTER] + Name lua + Match * + Script fluent-bit-script.lua + Call rewrite_body +[OUTPUT] + Name stdout + Match * + Format json +[OUTPUT] + Name http + Match * + Host oap + Port 12800 + URI /v3/logs + Format json diff --git a/test/e2e-v2/cases/log/h2/Dockerfile.h2 b/test/e2e-v2/cases/log/h2/Dockerfile.h2 new file mode 100644 index 000000000000..b77f407a55ff --- /dev/null +++ b/test/e2e-v2/cases/log/h2/Dockerfile.h2 @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM adoptopenjdk/openjdk8:alpine + +WORKDIR /h2 + +VOLUME /h2/data + +ADD https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200.jar /h2 + +CMD ["sh", "-c", "java -cp /h2/*.jar org.h2.tools.Server -tcp -tcpAllowOthers -tcpPort 1521 -ifNotExists -baseDir /h2/data"] \ No newline at end of file diff --git a/test/e2e-v2/cases/log/h2/docker-compose.yml b/test/e2e-v2/cases/log/h2/docker-compose.yml new file mode 100644 index 000000000000..6f657005c24c --- /dev/null +++ b/test/e2e-v2/cases/log/h2/docker-compose.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + h2db: + build: + context: . + dockerfile: Dockerfile.h2 + networks: + - e2e + expose: + - 1521 + healthcheck: + test: [ "CMD", "sh", "-c", "nc -z 127.0.0.1 1521" ] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: h2 + SW_STORAGE_H2_URL: jdbc:h2:tcp://h2db:1521/skywalking-oap-db + depends_on: + h2db: + condition: service_healthy + ports: + - 12800 + networks: + - e2e + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + networks: + - e2e + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/h2/e2e.yaml b/test/e2e-v2/cases/log/h2/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/h2/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file diff --git a/test/e2e-v2/cases/log/influxdb/docker-compose.yml b/test/e2e-v2/cases/log/influxdb/docker-compose.yml new file mode 100644 index 000000000000..5cc28269068b --- /dev/null +++ b/test/e2e-v2/cases/log/influxdb/docker-compose.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + influxdb: + image: influxdb:1.7.9 + expose: + - 8086 + networks: + - e2e + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8086"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: influxdb + SW_STORAGE_INFLUXDB_URL: "http://influxdb:8086" + ports: + - 12800 + networks: + - e2e + depends_on: + influxdb: + condition: service_healthy + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + networks: + - e2e + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/influxdb/e2e.yaml b/test/e2e-v2/cases/log/influxdb/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/influxdb/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file diff --git a/test/e2e-v2/cases/log/lal.yaml b/test/e2e-v2/cases/log/lal.yaml new file mode 100644 index 000000000000..bfd7a6105783 --- /dev/null +++ b/test/e2e-v2/cases/log/lal.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rules: + - name: example + dsl: | + filter { + text { + abortOnFailure false // for test purpose, we want to persist all logs + regexp $/(?s)(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}) \[TID:(?.+?)] \[(?.+?)] (?\w{4,}) (?.{1,36}) (?.+)/$ + } + extractor { + metrics { + timestamp log.timestamp as Long + labels level: parsed.level, service: log.service, instance: log.serviceInstance + name "log_count" + value 1 + } + } + sink { + } + } diff --git a/test/e2e-v2/cases/log/log-base-compose.yml b/test/e2e-v2/cases/log/log-base-compose.yml new file mode 100644 index 000000000000..8aebe75ddef1 --- /dev/null +++ b/test/e2e-v2/cases/log/log-base-compose.yml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + oap: + extends: + file: ../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_LOG_LAL_FILES: test + SW_LOG_MAL_FILES: test + volumes: + - ./lal.yaml:/skywalking/config/lal/test.yaml + - ./log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml + networks: + - e2e + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/log-mal.yaml b/test/e2e-v2/cases/log/log-mal.yaml new file mode 100644 index 000000000000..5f7f5ebcaa7b --- /dev/null +++ b/test/e2e-v2/cases/log/log-mal.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This will parse a textual representation of a duration. The formats +# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} +# with days considered to be exactly 24 hours. +#

+# Examples: +# 

+#    "PT20.345S" -- parses as "20.345 seconds"
+#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
+#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
+#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
+#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
+#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
+#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
+#
+ +expSuffix: instance(['service'], ['instance']) +metricPrefix: log +metricsRules: + - name: count_info + exp: log_count.tagEqual('level', 'INFO').sum(['service', 'instance']).downsampling(SUM) diff --git a/test/e2e-v2/cases/log/mysql/docker-compose.yml b/test/e2e-v2/cases/log/mysql/docker-compose.yml new file mode 100644 index 000000000000..8581a9249293 --- /dev/null +++ b/test/e2e-v2/cases/log/mysql/docker-compose.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + mysql: + image: mysql/mysql-server:8.0.13 + networks: + - e2e + expose: + - 3306 + environment: + MYSQL_ROOT_PASSWORD: "root@1234" + MYSQL_DATABASE: "swtest" + MYSQL_ROOT_HOST: "%" + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: mysql + SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest + ports: + - 12800 + entrypoint: ['sh', '-c', 'apk add --no-cache bash && /download-mysql.sh && /skywalking/docker-entrypoint.sh'] + networks: + - e2e + depends_on: + mysql: + condition: service_healthy + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + depends_on: + oap: + condition: service_healthy + ports: + - 9090 + networks: + - e2e + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/mysql/e2e.yaml b/test/e2e-v2/cases/log/mysql/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/mysql/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file diff --git a/test/e2e-v2/cases/log/postgres/docker-compose.yml b/test/e2e-v2/cases/log/postgres/docker-compose.yml new file mode 100644 index 000000000000..0c97992fd141 --- /dev/null +++ b/test/e2e-v2/cases/log/postgres/docker-compose.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + postgres: + image: postgres:13 + networks: + - e2e + expose: + - 5432 + environment: + - POSTGRES_PASSWORD=123456 + - POSTGRES_DB=skywalking + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../log-base-compose.yml + service: oap + environment: + SW_STORAGE: postgresql + SW_JDBC_URL: "jdbc:postgresql://postgres:5432/skywalking" + ports: + - 12800 + depends_on: + postgres: + condition: service_healthy + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + networks: + - e2e + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/log/postgres/e2e.yaml b/test/e2e-v2/cases/log/postgres/e2e.yaml new file mode 100644 index 000000000000..6f9824725036 --- /dev/null +++ b/test/e2e-v2/cases/log/postgres/e2e.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: ../expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: ../expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: ../expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: ../expected/logs.yml \ No newline at end of file