Merge pull request #15 from open-o11y/load-generator

Sig V4 Support in Prometheus Remote Write Exporter and OTLP Load generator for testing

Author: huyan0

Makefile

@@ -7,6 +7,7 @@ ALL_SRC := $(shell find . -name '*.go' \
 							-not -path '*/internal/data/opentelemetry-proto-gen/*' \
 							-not -path './.circleci/scripts/reportgenerator/*' \
 							-not -path './examples/demo/app/*' \
+							-not -path './test/*' \
 							-type f | sort)
 
 # ALL_PKGS is the list of all packages where ALL_SRC files reside.
@@ -31,6 +32,7 @@ STATIC_CHECK=staticcheck
 # BUILD_TYPE should be one of (dev, release).
 BUILD_TYPE?=release
 
+
 GIT_SHA=$(shell git rev-parse --short HEAD)
 BUILD_INFO_IMPORT_PATH=go.opentelemetry.io/collector/internal/version
 BUILD_X1=-X $(BUILD_INFO_IMPORT_PATH).GitHash=$(GIT_SHA)
@@ -49,6 +51,8 @@ all-srcs:
 
 all-pkgs:
 	@echo $(ALL_PKGS) | tr ' ' '\n' | sort
+all-env:
+	@printenv
 
 .DEFAULT_GOAL := all
 
@@ -156,7 +160,7 @@ install-tools:
 
 .PHONY: otelcol
 otelcol:
-	GO111MODULE=on CGO_ENABLED=0 go build -o ./bin/otelcol_$(GOOS)_$(GOARCH)$(EXTENSION) $(BUILD_INFO) ./cmd/otelcol
+	GO111MODULE=on CGO_ENABLED=0 go build -o ~/pipeline_custom/otelcol_$(GOOS)_$(GOARCH)$(EXTENSION) $(BUILD_INFO) ./cmd/otelcol
 
 .PHONY: run
 run:
@@ -296,3 +300,15 @@ check-contrib:
 	make -C $(CONTRIB_PATH) test
 	@echo Restoring contrib to no longer use this core checkout
 	make -C $(CONTRIB_PATH) for-all CMD="go mod edit -dropreplace go.opentelemetry.io/collector"
+
+.PHONY: testaps
+testaps: otelcol run-colletor-to-aps send-load
+
+.PHONY: run-colletor-to-aps
+run-colletor-to-aps:
+	GO111MODULE=on go run --race ./cmd/otelcol/... --config  ./test/otel-collector-config.yaml &
+
+.PHONY: send-load
+send-load:
+	@echo Starting OTLP load generator
+	go run ./test/otlploadgenerator/*.go

README.md

@@ -1,85 +1,85 @@
----
-
-<p align="center">
-  <strong>
-    <a href="https://opentelemetry.io/docs/collector/about/">Getting Started<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="https://github.com/open-telemetry/community#agentcollector">Getting Involved<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="https://gitter.im/open-telemetry/opentelemetry-service">Getting In Touch<a/>
-  </strong>
-</p>
-
-<p align="center">
-  <a href="https://goreportcard.com/report/github.com/open-telemetry/opentelemetry-collector">
-    <img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/open-telemetry/opentelemetry-collector?style=for-the-badge">
-  </a>
-  <a href="https://circleci.com/gh/open-telemetry/opentelemetry-collector">
-    <img alt="Build Status" src="https://img.shields.io/circleci/build/github/open-telemetry/opentelemetry-collector?style=for-the-badge">
-  </a>
-  <a href="https://codecov.io/gh/open-telemetry/opentelemetry-collector/branch/master/">
-    <img alt="Codecov Status" src="https://img.shields.io/codecov/c/github/open-telemetry/opentelemetry-collector?style=for-the-badge">
-  </a>
-  <a href="releases">
-    <img alt="GitHub release (latest by date including pre-releases)" src="https://img.shields.io/github/v/release/open-telemetry/opentelemetry-collector?include_prereleases&style=for-the-badge">
-  </a>
-  <img alt="Beta" src="https://img.shields.io/badge/status-beta-informational?style=for-the-badge&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAAAXNSR0IArs4c6QAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAACQAAAAAQAAAJAAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAABigAwAEAAAAAQAAABgAAAAA8A2UOAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAVlpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6dGlmZj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KTMInWQAABK5JREFUSA2dVm1sFEUYfmd2b/f2Pkqghn5eEQWKrRgjpkYgpoRCLC0oxV5apAiGUDEpJvwxEQ2raWPU+Kf8INU/RtEedwTCR9tYPloxGNJYTTQUwYqJ1aNpaLH3sXu3t7vjvFevpSqt7eSyM+/czvM8877PzB3APBoLgoDLsNePF56LBwqa07EKlDGg84CcWsI4CEbhNnDpAd951lXE2NkiNknCCTLv4HtzZuvPm1C/IKv4oDNXqNDHragety2XVzjECZsJARuBMyRzJrh1O0gQwLXuxofxsPSj4hG8fMLQo7bl9JJD8XZfC1E5yWFOMtd07dvX5kDwg6+2++Chq8txHGtfPoAp0gOFmhYoNFkHjn2TNUmrwRdna7W1QSkU8hvbGk4uThLrapaiLA2E6QY4u/lS9ItHfvJkxYsTMVtnAJLipYIWtVrcdX+8+b8IVnPl/R81prbuPZ1jpYw+0aEUGSkdFsgyBIaFTXCm6nyaxMtJ4n+TeDhJzGqZtQZcuYDgqDwDbqb0JF9oRpIG1Oea3bC1Y6N3x/WV8Zh83emhCs++hlaghDw+8w5UlYKq2lU7Pl8IkvS9KDqXmKmEwdMppVPKwGSEilmyAwJhRwWcq7wYC6z4wZ1rrEoMWxecdOjZWXeAQClBcYDN3NwVwD9pGwqUSyQgclcmxpNJqCuwLmDh3WtvPqXdlt+6Oz70HPGDNSNBee/EOen+rGbEFqDENBPDbtdCp0ukPANmzO0QQJYUpyS5IJJI3Hqt4maS+EB3199ozm8EDU/6fVNU2dQpdx3ZnKzeFXyaUTiasEV/gZMzJMjr3Z+WvAdQ+hs/zw9savimxUntDSaBdZ2f+Idbm1rlNY8esFffBit9HtK5/MejsrJVxikOXlb1Ukir2X+Rbdkd1KG2Ixfn2Ql4JRmELnYK9mEM8G36fAA3xEQ89fxXihC8q+sAKi9jhHxNqagY2hiaYgRCm0f0QP7H4Fp11LSXiuBY2aYFlh0DeDIVVFUJQn5rCnpiNI2gvLxHnASn9DIVHJJlm5rXvQAGEo4zvKq2w5G1NxENN7jrft1oxMdekETjxdH2Z3x+VTVYsPb+O0C/9/auN6v2hNZw5b2UOmSbG5/rkC3LBA+1PdxFxORjxpQ81GcxKc+ybVjEBvUJvaGJ7p7n5A5KSwe4AzkasA+crmzFtowoIVTiLjANm8GDsrWW35ScI3JY8Urv83tnkF8JR0yLvEt2hO/0qNyy3Jb3YKeHeHeLeOuVLRpNF+pkf85OW7/zJxWdXsbsKBUk2TC0BCPwMq5Q/CPvaJFkNS/1l1qUPe+uH3oD59erYGI/Y4sce6KaXYElAIOLt+0O3t2+/xJDF1XvOlWGC1W1B8VMszbGfOvT5qaRRAIFK3BCO164nZ0uYLH2YjNN8thXS2v2BK9gTfD7jHVxzHr4roOlEvYYz9QIz+Vl/sLDXInsctFsXjqIRnO2ZO387lxmIboLDZCJ59KLFliNIgh9ipt6tLg9SihpRPDO1ia5byw7de1aCQmF5geOQtK509rzfdwxaKOIq+73AvwCC5/5fcV4vo3+3LpMdtWHh0ywsJC/ZGoCb8/9D8F/ifgLLl8S8QWfU8cAAAAASUVORK5CYII=">
-</p>
-
-<p align="center">
-  <strong>
-    <a href="CONTRIBUTING.md">Contributing<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="docs/vision.md">Vision<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="docs/design.md">Design<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="docs/monitoring.md">Monitoring<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="docs/performance.md">Performance<a/>
-    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
-    <a href="docs/roadmap.md">Roadmap<a/>
-  </strong>
-</p>
-
----
-
-
-### OpenTelemetry Collector
-
-The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry data. In addition, it removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source telemetry data formats (e.g. Jaeger, Prometheus, etc.) sending to multiple open-source or commercial back-ends.
-
-Objectives:
-
-- Usable: Reasonable default configuration, supports popular protocols, runs and collects out of the box.
-- Performant: Highly stable and performant under varying loads and configurations.
-- Observable: An exemplar of an observable service.
-- Extensible: Customizable without touching the core code.
-- Unified: Single codebase, deployable as an agent or collector with support for traces, metrics, and logs (future).
-
-### Community Roles
-
-Triagers ([@open-telemetry/collector-triagers](https://github.com/orgs/open-telemetry/teams/collector-triager))
-- [Jay Camp](https://github.com/jrcamp), Splunk
-- [Steve Flanders](https://github.com/flands), Splunk
-
-Approvers ([@open-telemetry/collector-approvers](https://github.com/orgs/open-telemetry/teams/collector-approvers)):
-
-- [Dmitrii Anoshin](https://github.com/dmitryax), Splunk
-- [James Bebbington](https://github.com/james-bebbington), Google
-- [Nail Islamov](https://github.com/nilebox), Google
-- [Owais Lone](https://github.com/owais), Splunk
-
-Maintainers ([@open-telemetry/collector-maintainers](https://github.com/orgs/open-telemetry/teams/collector-maintainers)):
-
-- [Bogdan Drutu](https://github.com/BogdanDrutu), Splunk
-- [Paulo Janotti](https://github.com/pjanotti), Splunk
-- [Tigran Najaryan](https://github.com/tigrannajaryan), Splunk
-
-Learn more about roles in the [community repository](https://github.com/open-telemetry/community/blob/master/community-membership.md).
-
-Thanks to all the people who already contributed!
-
-<a href="https://github.com/open-telemetry/opentelemetry-collector/graphs/contributors">
-  <img src="https://contributors-img.web.app/image?repo=open-telemetry/opentelemetry-collector" />
-</a>
+# OpenTelemetry Collector O11y
+
+This repository is focused on building and packaging the OpenTelemetry Collector with a Prometheus remote write exporter
+ supporting Sig V4 to export to AWS services. See [Testing](#testing) section on how to generate and send a OTLP metric 
+ load to a 
+ Prometheus remote write backend.
+
+## Components
+
+Most upstream components are removed and not included in the build. Available components are:
+
+* Receiver: OpenTelemetry Protocol(OTLP) receiver 
+* Processor: None
+* Exporter: Logging exporter, Prometheus remote write exporter
+
+A Collector pipeline with these components is illustrated below:
+
+![Image: Repo README.png](./img/Pipeline.png)
+
+
+## Initialization
+
+Components in the collector are initialized by the Collector Service based on configuration. The factory and 
+configuration of a component are invoked at the initialization of the Collector Application, and they supply the 
+components with parameters components need. AWS Sig V4 support could be passed to the exporter by the factory without 
+impacting the data conversion and export logic. 
+
+The Prometheus remote write exporter use this pattern. During application initialization, the collector asks exporter 
+builders, a service package component, for an implementation of the MetricExporter interface. Then, the exporter builder
+ invokes the factory code inside the Prometheus Remote Write Exporter package to create an instance of the Prometheus 
+ Remote Write Exporter. When invoked, factory uses the exporter helper package to wrap the Exporter implementation 
+ inside the MetricExporter interface, and returns the instance to the Collector Application. **The factory can also 
+ pass in an [Sig V4 interceptor-attached http.Client](https://github.com/open-o11y/opentelemetry-collector/blob/d21a840612d40d935ccd52142f06e7106ee82384/exporter/prometheusremotewriteexporter/factory.go#L181)
+ to the exporter**.  Finally, the Collector Application
+  assembles the export pipeline with the exporter. During an export operation, the exporter behaves the same, but the 
+  interceptor-attached http.Client performs signing of each request.
+
+![Image: Class Diagram(2).png](./img/ClassDiagram.png)
+
+## Sample Configuration
+
+The following is a configuration for a Collector instance that receives gRPC OTLP metrics on `localhost:55680`, a 
+logging exporter that logs metric to `stdout`, and a Prometheus remote write exporter sending to an endpoint with AWS 
+Sig V4 support enabled. 
+
+```
+receivers:
+     otlp:
+      protocols:
+         grpc:
+exporters:
+  prometheusremotewrite:
+    endpoint: "https://aps-workspaces-beta.us-west-2.amazonaws.com/workspaces/yang-yu-intern-test-ws/remote_write"
+    namespace: otel-collector
+    auth:
+      region: "us-west-2"
+      service: "aps"
+    timeout: 10s
+  logging:
+    loglevel: debug
+
+service:
+  extensions:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      exporters: [logging,prometheusremotewrite]
+```
+
+see a complete list of configuration options and explanation of the prometheus remote write exporter [here](./exporter/prometheusremotewriteexporter/README.md)
+
+## Testing 
+
+To test the exporter, run the following command:
+
+```
+make testaps
+```
+
+This starts a Collector based on configuration in `/test/otel-config.yaml` and runs OTLP metric load generator to send 
+metrics to the exporter. OTLP metrics are generated based on `/test/otlploadgenerator/data.txt`. After the load generator exists,
+ it runs a querier to get metrics from the specified endpoint and write the output in `/test/querier/ans.txt`. 
+
+More details of testing, load generator and querier are described [here](./test/README.md). 
+

consumer/pdatautil/pdatautil.go

@@ -136,7 +136,6 @@ func CloneMetricsDataOld(md consumerdata.MetricsData) consumerdata.MetricsData {
 			clone.Metrics = append(clone.Metrics, metricClone)
 		}
 	}
-
 	return clone
 }