From b55ff6b44dec57ce6a288fc24d48b22944cbb445 Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sat, 18 Feb 2023 21:09:17 +0800 Subject: [PATCH 1/7] Support Amazon Simple Storage Service (Amazon S3) metrics monitoring --- .github/workflows/skywalking.yaml | 2 + docs/en/changes/changes.md | 1 + .../en/setup/backend/aws-firehose-receiver.md | 6 + .../backend/backend-aws-s3-monitoring.md | 37 +++ .../setup/backend/opentelemetry-receiver.md | 1 + docs/menu.yml | 2 + .../oap/server/core/analysis/Layer.java | 6 +- .../ui/template/UITemplateInitializer.java | 1 + .../aws/firehose/FirehoseHTTPHandler.java | 10 +- .../src/main/resources/application.yml | 2 +- .../otel-rules/aws-s3/s3-service.yaml | 55 ++++ .../aws_eks/aws_eks-root.json | 2 +- .../aws_s3/aws_s3-root.json | 68 ++++ .../aws_s3/aws_s3-service.json | 292 ++++++++++++++++++ test/e2e-v2/cases/aws/s3/docker-compose.yml | 51 +++ test/e2e-v2/cases/aws/s3/e2e.yaml | 44 +++ .../aws/s3/expected/metrics-has-value.yml | 19 ++ test/e2e-v2/cases/aws/s3/expected/service.yml | 24 ++ .../e2e-v2/cases/aws/s3/mock-data/data-1.json | 132 ++++++++ test/e2e-v2/cases/aws/s3/s3-cases.yaml | 28 ++ 20 files changed, 778 insertions(+), 5 deletions(-) create mode 100644 docs/en/setup/backend/backend-aws-s3-monitoring.md create mode 100644 oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml create mode 100644 oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-root.json create mode 100644 oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json create mode 100644 test/e2e-v2/cases/aws/s3/docker-compose.yml create mode 100644 test/e2e-v2/cases/aws/s3/e2e.yaml create mode 100644 test/e2e-v2/cases/aws/s3/expected/metrics-has-value.yml create mode 100644 test/e2e-v2/cases/aws/s3/expected/service.yml create mode 100644 test/e2e-v2/cases/aws/s3/mock-data/data-1.json create mode 100644 test/e2e-v2/cases/aws/s3/s3-cases.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index fe6e025d684c..e74bbae57a40 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -670,6 +670,8 @@ jobs: config: test/e2e-v2/cases/aws/eks/e2e.yaml - name: Windows config: test/e2e-v2/cases/win/e2e.yaml + - name: AWS Cloud S3 + config: test/e2e-v2/cases/aws/s3/e2e.yaml steps: - uses: actions/checkout@v3 with: diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 153b33de3cfa..47f800f617d5 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -95,6 +95,7 @@ * Add Python HTTPX module component ID(7019). * Migrate tests from junit 4 to junit 5. * Refactor http-based alarm plugins and extract common logic to `HttpAlarmCallback`. +* Support Amazon Simple Storage Service (Amazon S3) metrics monitoring #### UI diff --git a/docs/en/setup/backend/aws-firehose-receiver.md b/docs/en/setup/backend/aws-firehose-receiver.md index e422c46f4da7..254a3c7f81eb 100644 --- a/docs/en/setup/backend/aws-firehose-receiver.md +++ b/docs/en/setup/backend/aws-firehose-receiver.md @@ -14,6 +14,12 @@ Usually, the [AWS CloudWatch metrics](https://docs.aws.amazon.com/AmazonCloudWat CloudWatch metrics with S3 --> CloudWatch Metric Stream (OpenTelemetry formart) --> Kinesis Data Firehose Delivery Stream --> AWS Firehose receiver(OAP) --> OpenTelemetry receiver(OAP) ``` +## Supported metrics + +| Description | Configuration File | Data Source | +|-----------------------------------------|-------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| Metrics of AWS Cloud S3 | otel-rules/aws-s3/s3-service.yaml | AWS CloudWatcher Metrics Stream -> AWS Firehose delivery stream -> SkyWalking OAP Server with [AWS Firehose receiver](./aws-firehose-receiver.md) | + ## Notice 1. Only OpenTelemetry format is supported (refer to [Metric streams output formats](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats.html)) diff --git a/docs/en/setup/backend/backend-aws-s3-monitoring.md b/docs/en/setup/backend/backend-aws-s3-monitoring.md new file mode 100644 index 000000000000..c34292974d42 --- /dev/null +++ b/docs/en/setup/backend/backend-aws-s3-monitoring.md @@ -0,0 +1,37 @@ +# AWS Cloud S3 monitoring +Amazon Simple Storage Service (Amazon S3) is an object storage service. SkyWalking leverages [AWS Kinesis Data Firehose receiver](./aws-firehose-receiver.md) to transfer the CloudWatch metrics of s3 to +[OpenTelemetry receiver](opentelemetry-receiver.md) and into the [Meter System](./../../concepts-and-designs/meter.md). + +### Data flow +1. AWS CloudWatch collect metrics for S3, refer to [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) +2. [CloudWatch metric streams](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html) stream CloudWatch metrics of S3 to AWS Kinesis Data Firehose +3. AWS Kinesis Data Firehose delivery metrics to [AWS Kinesis Data Firehose receiver](./aws-firehose-receiver.md) through the HTTP endpoint + +### Set up +1. Create CloudWatch metrics configuration for S3, refer to [S3 metrics configuration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/metrics-configurations.html) +2. Create an Amazon Kinesis Data Firehose Delivery Stream, and set [AWS Kinesis Data Firehose receiver](./aws-firehose-receiver.md)'s address as HTTP Destination, refer to [Create Delivery Stream](https://docs.aws.amazon.com/firehose/latest/dev/basic-create.html) +3. Create CloudWatch metric stream, and select the Firehose Delivery Stream which has been created above, set `Select namespaces` to `AWS/S3`, `Select output format` to `OpenTelemetry 0.7`. refer to [CloudWatch Metric Streams](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html) + +### S3 Monitoring + +SkyWalking observes CloudWatch metrics of the S3 bucket, which is cataloged as a `LAYER: AWS_S3` `Service` in the OAP. + +#### Supported Metrics + +| Monitoring Panel | Unit | Metric Name | Catalog | Description | Data Source | +|----------------------------|-------|-----------------------------|----------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| 4xx Errors | count | aws_s3_4xx | Service | The number of HTTP 4xx client error status code requests made to the S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| 5xx Errors | count | aws_s3_5xx | Service | The number of HTTP 5xx client error status code requests made to the S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Downloaded | bytes | aws_s3_downloaded_bytes | Service | The number of bytes downloaded for requests made to an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Uploaded | bytes | aws_s3_uploaded_bytes | Service | The number of bytes uploaded for requests made to an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Request Average Latency | bytes | aws_s3_request_latency | Service | The average of elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| First Byte Average Latency | bytes | aws_s3_request_latency | Service | The average of per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| All Requests | bytes | aws_s3_delete_requests | Service | The number of HTTP All requests made for objects in an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Get Requests | bytes | aws_s3_delete_requests | Service | The number of HTTP Get requests made for objects in an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Put Requests | bytes | aws_s3_delete_requests | Service | The number of HTTP PUT requests made for objects in an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | +| Delete Requests | bytes | aws_s3_delete_requests | Service | The number of HTTP Delete requests made for objects in an Amazon S3 bucket | [S3 monitoring with CloudWatch](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cloudwatch-monitoring.html) | + +### Customizations +You can customize your own metrics/expression/dashboard panel. +The metrics definition and expression rules are found in `/config/otel-rules/aws-s3/`. +The AWS Cloud EKS dashboard panel configurations are found in `/config/ui-initialized-templates/aws_s3`. diff --git a/docs/en/setup/backend/opentelemetry-receiver.md b/docs/en/setup/backend/opentelemetry-receiver.md index 41acc4cd3f67..348e5c850cb7 100644 --- a/docs/en/setup/backend/opentelemetry-receiver.md +++ b/docs/en/setup/backend/opentelemetry-receiver.md @@ -43,5 +43,6 @@ for identification of the metric data. | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-cluster.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-service.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-node.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | +| Metrics of AWS Cloud S3 | otel-rules/aws-s3/s3-service.yaml | AWS CloudWatcher Metrics Stream -> AWS Firehose delivery stream -> SkyWalking OAP Server with [AWS Firehose receiver](./aws-firehose-receiver.md) | **Note**: You can also use OpenTelemetry exporter to transport the metrics to SkyWalking OAP directly. See [OpenTelemetry Exporter](./backend-meter.md#opentelemetry-exporter). diff --git a/docs/menu.yml b/docs/menu.yml index 1a74b095b762..0882f8bab94d 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -189,6 +189,8 @@ catalog: catalog: - name: "EKS Monitoring" path: "/en/setup/backend/backend-aws-eks-monitoring" + - name: "S3 Monitoring" + path: "/en/setup/backend/backend-aws-s3-monitoring" - name: "Browser Monitoring" path: "/en/setup/service-agent/browser-agent" - name: "Gateway Monitoring" diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/Layer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/Layer.java index dd4da5ad0592..a939c5765341 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/Layer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/Layer.java @@ -146,8 +146,12 @@ public enum Layer { /** * Windows Machine */ - OS_WINDOWS(23, true); + OS_WINDOWS(23, true), + /** + * Amazon Simple Storage Service (Amazon S3) is an object storage service provided by AWS Cloud + */ + AWS_S3(24, true); private final int value; /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java index 49539ff305a3..4ec3ac58c7f7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java @@ -62,6 +62,7 @@ public class UITemplateInitializer { Layer.VIRTUAL_MQ.name(), Layer.AWS_EKS.name(), Layer.OS_WINDOWS.name(), + Layer.AWS_S3.name(), "custom" }; private final UITemplateManagementService uiTemplateManagementService; diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java index 9a11018a942f..496cc5785ae9 100644 --- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java +++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java @@ -17,7 +17,9 @@ package org.apache.skywalking.oap.server.receiver.aws.firehose; +import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.server.annotation.ConsumesJson; @@ -33,20 +35,24 @@ @Slf4j @AllArgsConstructor public class FirehoseHTTPHandler { + private final OpenTelemetryMetricRequestProcessor openTelemetryMetricRequestProcessor; @Post("/aws/firehose/metrics") @ConsumesJson @ProducesJson public HttpResponse collectMetrics(final FirehoseReq firehoseReq) { + log.info("receiver--> " + new Gson().toJson(firehoseReq)); try { for (RequestData record : firehoseReq.getRecords()) { final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( Base64.getDecoder().decode(record.getData())); ExportMetricsServiceRequest request; while ((request = ExportMetricsServiceRequest.parseDelimitedFrom(byteArrayInputStream)) != null) { - openTelemetryMetricRequestProcessor.processMetricsRequest( - OtelMetricsConvertor.convertExportMetricsRequest(request)); + final io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest target = OtelMetricsConvertor.convertExportMetricsRequest( + request); + log.info("receiver otel -> " + JsonFormat.printer().print(target)); + openTelemetryMetricRequestProcessor.processMetricsRequest(target); } } } catch (InvalidProtocolBufferException e) { diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index a08843be1d9c..a42c55bd2645 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -356,7 +356,7 @@ receiver-otel: selector: ${SW_OTEL_RECEIVER:default} default: enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"oc,otlp"} - enabledOtelRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_RULES:"apisix,k8s/*,istio-controlplane,vm,mysql/*,postgresql/*,oap,aws-eks/*,windows"} + enabledOtelRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_RULES:"apisix,k8s/*,istio-controlplane,vm,mysql/*,postgresql/*,oap,aws-eks/*,windows,aws-s3/*"} receiver-zipkin: selector: ${SW_RECEIVER_ZIPKIN:-} diff --git a/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml b/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml new file mode 100644 index 000000000000..2c3a5b6d0814 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This will parse a textual representation of a duration. The formats +# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS} +# with days considered to be exactly 24 hours. +#

+# Examples: +# 

+#    "PT20.345S" -- parses as "20.345 seconds"
+#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
+#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
+#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
+#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
+#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
+#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
+#
+ +filter: "{ tags -> {tags.cloud_provider == 'aws' && tags.Namespace == 'AWS/S3' } }" +expSuffix: tag({tags -> tags.bucket = 'aws-s3-buckets::' + tags.BucketName}).service(['bucket'], Layer.AWS_S3) +metricPrefix: aws_s3 +metricsRules: + - name: 4xx + exp: amazonaws_com_AWS_S3_4xxErrors_sum.downsampling(SUM) + - name: 5xx + exp: amazonaws_com_AWS_S3_5xxErrors_sum.downsampling(SUM) + - name: downloaded_bytes + exp: amazonaws_com_AWS_S3_BytesDownloaded_sum.downsampling(SUM) + - name: uploaded_bytes + exp: amazonaws_com_AWS_S3_BytesUploaded_sum.downsampling(SUM) + - name: request_latency + exp: amazonaws_com_AWS_S3_TotalRequestLatency_sum.div(amazonaws_com_AWS_S3_TotalRequestLatency_count) + - name: first_latency_bytes + exp: amazonaws_com_AWS_S3_FirstByteLatency_sum.div(amazonaws_com_AWS_S3_FirstByteLatency_count) + - name: all_requests + exp: amazonaws_com_AWS_S3_AllRequests_sum.downsampling(SUM) + - name: put_requests + exp: amazonaws_com_AWS_S3_PutRequests_sum.downsampling(SUM) + - name: get_requests + exp: amazonaws_com_AWS_S3_GetRequests_sum.downsampling(SUM) + - name: delete_requests + exp: amazonaws_com_AWS_S3_DeleteRequests_sum.downsampling(SUM) diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_eks/aws_eks-root.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_eks/aws_eks-root.json index 401538bbc772..d5e8206817eb 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_eks/aws_eks-root.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_eks/aws_eks-root.json @@ -50,7 +50,7 @@ "graph": { "fontColor":"blue", "backgroundColor":"white", - "content":"Provide AWS Could monitoring through AWS Container Insights Receiver", + "content":"Provide AWS Cloud EKS monitoring through AWS Container Insights Receiver", "fontSize":14, "textAlign":"left", "url":"https://skywalking.apache.org/docs/main/next/en/setup/backend/backend-aws-eks-monitoring/" diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-root.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-root.json new file mode 100644 index 000000000000..459071991fe2 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-root.json @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +[ + { + "id": "AWS_S3-Root", + "configuration": { + "children": [ + { + "x": 0, + "y": 2, + "w": 24, + "h": 51, + "i": "0", + "type": "Widget", + "widget": { + "title": "" + }, + "graph": { + "type": "ServiceList", + "dashboardName": "AWS_S3-service", + "fontSize": 12, + "showXAxis": false, + "showYAxis": false, + "showGroup": true + }, + "moved": false + }, + { + "x": 0, + "y": 0, + "w": 24, + "h": 2, + "i": "100", + "type": "Text", + "graph": { + "fontColor":"blue", + "backgroundColor":"white", + "content":"Provide AWS Cloud S3 monitoring through AWS FireHose Receiver", + "fontSize":14, + "textAlign":"left", + "url":"https://skywalking.apache.org/docs/main/next/en/setup/backend/backend-aws-s3-monitoring/" + }, + "moved": false + } + ], + "id": "AWS_S3-Root", + "layer": "AWS_S3", + "entity": "All", + "name": "AWS_S3-Root", + "isRoot": true + } + } +] diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json new file mode 100644 index 000000000000..845e3eaceedf --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +[ + { + "id": "AWS_S3-service", + "configuration": { + "children": [ + { + "x": 0, + "y": 0, + "w": 6, + "h": 13, + "i": "3", + "type": "Widget", + "widget": { + "title": "Downloaded (KB)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_downloaded_bytes" + ], + "metricTypes": [ + "readMetricsValues" + ], + "metricConfig": [ + { + "calculation": "byteToKB" + } + ] + }, + { + "x": 6, + "y": 0, + "w": 6, + "h": 13, + "i": "4", + "type": "Widget", + "widget": { + "title": "Uploaded (KB)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_uploaded_bytes" + ], + "metricTypes": [ + "readMetricsValues" + ], + "metricConfig": [ + { + "calculation": "byteToKB" + } + ] + }, + { + "x": 12, + "y": 0, + "w": 6, + "h": 13, + "i": "5", + "type": "Widget", + "widget": { + "title": "4xx Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_4xx" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 18, + "y": 0, + "w": 6, + "h": 13, + "i": "6", + "type": "Widget", + "widget": { + "title": "5xx Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_4xx" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 0, + "y": 13, + "w": 6, + "h": 13, + "i": "7", + "type": "Widget", + "widget": { + "title": "Request Average Latency (ms)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_request_latency" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 6, + "y": 13, + "w": 6, + "h": 13, + "i": "8", + "type": "Widget", + "widget": { + "title": "First Byte Average Latency (ms)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_first_latency_bytes" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 12, + "y": 13, + "w": 6, + "h": 13, + "i": "9", + "type": "Widget", + "widget": { + "title": "All Request Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_all_requests" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 18, + "y": 13, + "w": 6, + "h": 13, + "i": "10", + "type": "Widget", + "widget": { + "title": "Get Request Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_get_requests" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 0, + "y": 26, + "w": 6, + "h": 13, + "i": "11", + "type": "Widget", + "widget": { + "title": "Put Request Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_put_requests" + ], + "metricTypes": [ + "readMetricsValues" + ] + }, + { + "x": 6, + "y": 26, + "w": 6, + "h": 13, + "i": "12", + "type": "Widget", + "widget": { + "title": "Delete Request Count" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true + }, + "metrics": [ + "aws_s3_delete_requests" + ], + "metricTypes": [ + "readMetricsValues" + ] + } + ], + + "layer": "AWS_S3", + "entity": "Service", + "name": "AWS_S3-service", + "id": "AWS_S3-service", + "isRoot": false + } + } +] diff --git a/test/e2e-v2/cases/aws/s3/docker-compose.yml b/test/e2e-v2/cases/aws/s3/docker-compose.yml new file mode 100644 index 000000000000..dcd2cfc5af6f --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/docker-compose.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + ports: + - 12800 + networks: + e2e: + + sender: + image: "eclipse-temurin:8-jre" + volumes: + - ./../../../java-test-service/e2e-mock-sender/target/e2e-mock-sender-2.0.0.jar:/e2e-mock-sender-2.0.0.jar + - ./mock-data:/data/otel-metrics + command: [ "java", "-jar", "/e2e-mock-sender-2.0.0.jar" ] + environment: + OAP_HOST: oap + OAP_GRPC_PORT: 11800 + OTEL_METRICS_DATA_PATH: /data/otel-metrics + networks: + - e2e + ports: + - 9093:9093 + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9093"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy +networks: + e2e: diff --git a/test/e2e-v2/cases/aws/s3/e2e.yaml b/test/e2e-v2/cases/aws/s3/e2e.yaml new file mode 100644 index 000000000000..5cde5b61d16b --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/e2e.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://localhost:9093/otel-metrics/send + method: GET + +verify: + retry: + count: 10 + interval: 3s + cases: + - includes: + - ./s3-cases.yaml diff --git a/test/e2e-v2/cases/aws/s3/expected/metrics-has-value.yml b/test/e2e-v2/cases/aws/s3/expected/metrics-has-value.yml new file mode 100644 index 000000000000..ce1f3d83c82f --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/expected/metrics-has-value.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- key: {{ notEmpty .key }} + value: {{ ge .value 0 }} +{{- end }} diff --git a/test/e2e-v2/cases/aws/s3/expected/service.yml b/test/e2e-v2/cases/aws/s3/expected/service.yml new file mode 100644 index 000000000000..dfb982bc7877 --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/expected/service.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + {{- contains . }} +- id: {{ b64enc "aws-s3-buckets::skywalking" }}.1 + name: aws-s3-buckets::skywalking + group: aws-s3-buckets + shortname: skywalking + normal: true + layers: + - AWS_S3 + {{- end }} diff --git a/test/e2e-v2/cases/aws/s3/mock-data/data-1.json b/test/e2e-v2/cases/aws/s3/mock-data/data-1.json new file mode 100644 index 000000000000..7d97bca7bf96 --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/mock-data/data-1.json @@ -0,0 +1,132 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "cloud.provider", + "value": { + "stringValue": "aws" + } + }, + { + "key": "cloud.account.id", + "value": { + "stringValue": "xxxxxxxx" + } + }, + { + "key": "cloud.region", + "value": { + "stringValue": "ap-northeast-1" + } + }, + { + "key": "aws.exporter.arn", + "value": { + "stringValue": "arn:aws:cloudwatch:ap-northeast-1:xxxxxxxx:metric-stream/CustomFull-CmHV2P" + } + } + ] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": "amazonaws.com/AWS/S3/4xxErrors", + "unit": "{Count}", + "summary": { + "dataPoints": [ + { + "startTimeUnixNano": "1674547500000000000", + "timeUnixNano": "1674547560000000000", + "count": "1", + "quantileValues": [ + { + }, + { + "quantile": 1.0 + } + ], + "attributes": [ + { + "key": "Namespace", + "value": { + "stringValue": "AWS/S3" + } + }, + { + "key": "MetricName", + "value": { + "stringValue": "4xxErrors" + } + }, + { + "key": "BucketName", + "value": { + "stringValue": "skywalking" + } + }, + { + "key": "FilterId", + "value": { + "stringValue": "test" + } + } + ] + } + ] + } + }, + { + "name": "amazonaws.com/AWS/S3/5xxErrors", + "unit": "{Count}", + "summary": { + "dataPoints": [ + { + "startTimeUnixNano": "1674547500000000000", + "timeUnixNano": "1674547560000000000", + "count": "1", + "quantileValues": [ + { + }, + { + "quantile": 1.0 + } + ], + "attributes": [ + { + "key": "Namespace", + "value": { + "stringValue": "AWS/S3" + } + }, + { + "key": "MetricName", + "value": { + "stringValue": "5xxErrors" + } + }, + { + "key": "BucketName", + "value": { + "stringValue": "skywalking" + } + }, + { + "key": "FilterId", + "value": { + "stringValue": "test" + } + } + ] + } + ] + } + } + ] + } + ] + } + ] +} diff --git a/test/e2e-v2/cases/aws/s3/s3-cases.yaml b/test/e2e-v2/cases/aws/s3/s3-cases.yaml new file mode 100644 index 000000000000..f769971d139e --- /dev/null +++ b/test/e2e-v2/cases/aws/s3/s3-cases.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + + cases: + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear \ + --name=aws_s3_4xx --service-name aws-s3-buckets::skywalking |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear \ + --name=aws_s3_5xx --service-name aws-s3-buckets::skywalking |yq e 'to_entries' - + expected: expected/metrics-has-value.yml From 7a189c4a19071df8f35a5a14a1104fad7b28e56f Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sat, 18 Feb 2023 21:18:53 +0800 Subject: [PATCH 2/7] revert FirehoseHTTPHandler.java --- .../receiver/aws/firehose/FirehoseHTTPHandler.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java index 496cc5785ae9..9a11018a942f 100644 --- a/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java +++ b/oap-server/server-receiver-plugin/aws-firehose-receiver/src/main/java/org/apache/skywalking/oap/server/receiver/aws/firehose/FirehoseHTTPHandler.java @@ -17,9 +17,7 @@ package org.apache.skywalking.oap.server.receiver.aws.firehose; -import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.server.annotation.ConsumesJson; @@ -35,24 +33,20 @@ @Slf4j @AllArgsConstructor public class FirehoseHTTPHandler { - private final OpenTelemetryMetricRequestProcessor openTelemetryMetricRequestProcessor; @Post("/aws/firehose/metrics") @ConsumesJson @ProducesJson public HttpResponse collectMetrics(final FirehoseReq firehoseReq) { - log.info("receiver--> " + new Gson().toJson(firehoseReq)); try { for (RequestData record : firehoseReq.getRecords()) { final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( Base64.getDecoder().decode(record.getData())); ExportMetricsServiceRequest request; while ((request = ExportMetricsServiceRequest.parseDelimitedFrom(byteArrayInputStream)) != null) { - final io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest target = OtelMetricsConvertor.convertExportMetricsRequest( - request); - log.info("receiver otel -> " + JsonFormat.printer().print(target)); - openTelemetryMetricRequestProcessor.processMetricsRequest(target); + openTelemetryMetricRequestProcessor.processMetricsRequest( + OtelMetricsConvertor.convertExportMetricsRequest(request)); } } } catch (InvalidProtocolBufferException e) { From 2f6f4342929d75320efcaf6aba8e79f8cc6b089b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Sat, 18 Feb 2023 21:32:58 +0800 Subject: [PATCH 3/7] Update docs/en/setup/backend/backend-aws-s3-monitoring.md --- docs/en/setup/backend/backend-aws-s3-monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/setup/backend/backend-aws-s3-monitoring.md b/docs/en/setup/backend/backend-aws-s3-monitoring.md index c34292974d42..390502be7969 100644 --- a/docs/en/setup/backend/backend-aws-s3-monitoring.md +++ b/docs/en/setup/backend/backend-aws-s3-monitoring.md @@ -9,7 +9,7 @@ Amazon Simple Storage Service (Amazon S3) is an object storage service. SkyWalki ### Set up 1. Create CloudWatch metrics configuration for S3, refer to [S3 metrics configuration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/metrics-configurations.html) -2. Create an Amazon Kinesis Data Firehose Delivery Stream, and set [AWS Kinesis Data Firehose receiver](./aws-firehose-receiver.md)'s address as HTTP Destination, refer to [Create Delivery Stream](https://docs.aws.amazon.com/firehose/latest/dev/basic-create.html) +2. Create an Amazon Kinesis Data Firehose Delivery Stream, and set [AWS Kinesis Data Firehose receiver](./aws-firehose-receiver.md)'s address as HTTP(s) Destination, refer to [Create Delivery Stream](https://docs.aws.amazon.com/firehose/latest/dev/basic-create.html) 3. Create CloudWatch metric stream, and select the Firehose Delivery Stream which has been created above, set `Select namespaces` to `AWS/S3`, `Select output format` to `OpenTelemetry 0.7`. refer to [CloudWatch Metric Streams](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html) ### S3 Monitoring From 456f2a39041df488fa34d771c06f3902fa973de5 Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sat, 18 Feb 2023 21:38:42 +0800 Subject: [PATCH 4/7] revert opentelemetry-receiver.md --- docs/en/setup/backend/opentelemetry-receiver.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/setup/backend/opentelemetry-receiver.md b/docs/en/setup/backend/opentelemetry-receiver.md index 348e5c850cb7..41acc4cd3f67 100644 --- a/docs/en/setup/backend/opentelemetry-receiver.md +++ b/docs/en/setup/backend/opentelemetry-receiver.md @@ -43,6 +43,5 @@ for identification of the metric data. | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-cluster.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-service.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | | Metrics of AWS Cloud EKS| otel-rules/aws-eks/eks-node.yaml |AWS Container Insights Receiver -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server | -| Metrics of AWS Cloud S3 | otel-rules/aws-s3/s3-service.yaml | AWS CloudWatcher Metrics Stream -> AWS Firehose delivery stream -> SkyWalking OAP Server with [AWS Firehose receiver](./aws-firehose-receiver.md) | **Note**: You can also use OpenTelemetry exporter to transport the metrics to SkyWalking OAP directly. See [OpenTelemetry Exporter](./backend-meter.md#opentelemetry-exporter). From 7b75b40ff7beeb2055b9c475bdffbe2f628e9857 Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sun, 19 Feb 2023 00:21:39 +0800 Subject: [PATCH 5/7] Update oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 吴晟 Wu Sheng --- .../src/main/resources/otel-rules/aws-s3/s3-service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml b/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml index 2c3a5b6d0814..6cfbf1718c7f 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/aws-s3/s3-service.yaml @@ -30,7 +30,7 @@ # filter: "{ tags -> {tags.cloud_provider == 'aws' && tags.Namespace == 'AWS/S3' } }" -expSuffix: tag({tags -> tags.bucket = 'aws-s3-buckets::' + tags.BucketName}).service(['bucket'], Layer.AWS_S3) +expSuffix: tag({tags -> tags.bucket = 'aws-s3::' + tags.BucketName}).service(['bucket'], Layer.AWS_S3) metricPrefix: aws_s3 metricsRules: - name: 4xx From 9db1b95be12cb7f99ad8dcd23a3410d1c9308322 Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sun, 19 Feb 2023 14:01:28 +0800 Subject: [PATCH 6/7] update s3 test case -> aws-s3::skywalking --- test/e2e-v2/cases/aws/s3/expected/service.yml | 6 +++--- test/e2e-v2/cases/aws/s3/s3-cases.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e-v2/cases/aws/s3/expected/service.yml b/test/e2e-v2/cases/aws/s3/expected/service.yml index dfb982bc7877..f6021171009b 100644 --- a/test/e2e-v2/cases/aws/s3/expected/service.yml +++ b/test/e2e-v2/cases/aws/s3/expected/service.yml @@ -14,9 +14,9 @@ # limitations under the License. {{- contains . }} -- id: {{ b64enc "aws-s3-buckets::skywalking" }}.1 - name: aws-s3-buckets::skywalking - group: aws-s3-buckets +- id: {{ b64enc "aws-s3::skywalking" }}.1 + name: aws-s3::skywalking + group: aws-s3 shortname: skywalking normal: true layers: diff --git a/test/e2e-v2/cases/aws/s3/s3-cases.yaml b/test/e2e-v2/cases/aws/s3/s3-cases.yaml index f769971d139e..1c6ebb8c87a2 100644 --- a/test/e2e-v2/cases/aws/s3/s3-cases.yaml +++ b/test/e2e-v2/cases/aws/s3/s3-cases.yaml @@ -20,9 +20,9 @@ expected: expected/service.yml - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear \ - --name=aws_s3_4xx --service-name aws-s3-buckets::skywalking |yq e 'to_entries' - + --name=aws_s3_4xx --service-name aws-s3::skywalking |yq e 'to_entries' - expected: expected/metrics-has-value.yml - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear \ - --name=aws_s3_5xx --service-name aws-s3-buckets::skywalking |yq e 'to_entries' - + --name=aws_s3_5xx --service-name aws-s3::skywalking |yq e 'to_entries' - expected: expected/metrics-has-value.yml From 860b6e2b28214639de7bdf6f50dfc45644a7a324 Mon Sep 17 00:00:00 2001 From: "pg.yang" Date: Sun, 19 Feb 2023 14:17:37 +0800 Subject: [PATCH 7/7] update layer --- .../aws_s3/aws_s3-service.json | 201 ++++++++++++++---- 1 file changed, 154 insertions(+), 47 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json index 845e3eaceedf..6c4a39279946 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/aws_s3/aws_s3-service.json @@ -24,66 +24,98 @@ "x": 0, "y": 0, "w": 6, - "h": 13, - "i": "3", + "h": 5, + "i": "1", "type": "Widget", "widget": { - "title": "Downloaded (KB)" + "title": "4xx Count" }, "graph": { - "type": "Line", - "step": false, - "smooth": false, - "showSymbol": false, - "showXAxis": true, - "showYAxis": true + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUnit": true }, "metrics": [ - "aws_s3_downloaded_bytes" + "aws_s3_4xx" ], "metricTypes": [ - "readMetricsValues" - ], - "metricConfig": [ - { - "calculation": "byteToKB" - } + "readMetricsValue" ] }, { "x": 6, "y": 0, "w": 6, - "h": 13, - "i": "4", + "h": 5, + "i": "2", "type": "Widget", "widget": { - "title": "Uploaded (KB)" + "title": "5xx Count" }, "graph": { - "type": "Line", - "step": false, - "smooth": false, - "showSymbol": false, - "showXAxis": true, - "showYAxis": true + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUnit": true }, "metrics": [ - "aws_s3_uploaded_bytes" + "aws_s3_4xx" ], "metricTypes": [ - "readMetricsValues" - ], - "metricConfig": [ - { - "calculation": "byteToKB" - } + "readMetricsValue" ] }, { "x": 12, "y": 0, "w": 6, + "h": 5, + "i": "3", + "type": "Widget", + "widget": { + "title": "All Request Count" + }, + "graph": { + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUnit": true + }, + "metrics": [ + "aws_s3_all_requests" + ], + "metricTypes": [ + "readMetricsValue" + ] + }, + { + "x": 18, + "y": 0, + "w": 6, + "h": 5, + "i": "4", + "type": "Widget", + "widget": { + "title": "Request Average Latency (ms)" + }, + "graph": { + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUnit": true + }, + "metrics": [ + "aws_s3_request_latency" + ], + "metricTypes": [ + "readMetricsValue" + ] + }, + { + "x": 0, + "y": 5, + "w": 6, "h": 13, "i": "5", "type": "Widget", @@ -106,8 +138,8 @@ ] }, { - "x": 18, - "y": 0, + "x": 6, + "y": 5, "w": 6, "h": 13, "i": "6", @@ -131,8 +163,8 @@ ] }, { - "x": 0, - "y": 13, + "x": 12, + "y": 5, "w": 6, "h": 13, "i": "7", @@ -156,8 +188,8 @@ ] }, { - "x": 6, - "y": 13, + "x": 18, + "y": 5, "w": 6, "h": 13, "i": "8", @@ -181,8 +213,8 @@ ] }, { - "x": 12, - "y": 13, + "x": 0, + "y": 18, "w": 6, "h": 13, "i": "9", @@ -206,8 +238,8 @@ ] }, { - "x": 18, - "y": 13, + "x": 6, + "y": 18, "w": 6, "h": 13, "i": "10", @@ -231,8 +263,8 @@ ] }, { - "x": 0, - "y": 26, + "x": 12, + "y": 18, "w": 6, "h": 13, "i": "11", @@ -256,8 +288,8 @@ ] }, { - "x": 6, - "y": 26, + "x": 18, + "y": 18, "w": 6, "h": 13, "i": "12", @@ -279,9 +311,84 @@ "metricTypes": [ "readMetricsValues" ] + }, + { + "x": 0, + "y": 41, + "w": 12, + "h": 13, + "i": "13", + "type": "Widget", + "widget": { + "title": "Downloaded (KB)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true, + "legend": { + "asTable": true, + "toTheRight": true, + "min": true, + "max": true, + "mean": true, + "total": true + } + }, + "metrics": [ + "aws_s3_downloaded_bytes" + ], + "metricTypes": [ + "readMetricsValues" + ], + "metricConfig": [ + { + "calculation": "byteToKB" + } + ] + }, + { + "x": 12, + "y": 41, + "w": 12, + "h": 13, + "i": "14", + "type": "Widget", + "widget": { + "title": "Uploaded (KB)" + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": false, + "showXAxis": true, + "showYAxis": true, + "legend": { + "asTable": true, + "toTheRight": true, + "min": true, + "max": true, + "mean": true, + "total": true + } + }, + "metrics": [ + "aws_s3_uploaded_bytes" + ], + "metricTypes": [ + "readMetricsValues" + ], + "metricConfig": [ + { + "calculation": "byteToKB" + } + ] } ], - "layer": "AWS_S3", "entity": "Service", "name": "AWS_S3-service",