From c6868fde297c6b6fe2883d7223417840808495be Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Wed, 29 Jul 2020 10:03:46 -0700 Subject: [PATCH 01/15] Rename exporter packages from "ext" to "exporter" (#953) --- .../opentelemetry-exporter-otlp/CHANGELOG.md | 18 + exporter/opentelemetry-exporter-otlp/LICENSE | 201 +++++++++++ .../opentelemetry-exporter-otlp/MANIFEST.in | 9 + .../opentelemetry-exporter-otlp/README.rst | 25 ++ .../opentelemetry-exporter-otlp/setup.cfg | 54 +++ exporter/opentelemetry-exporter-otlp/setup.py | 26 ++ .../opentelemetry/exporter/otlp/__init__.py | 57 +++ .../exporter/otlp/trace_exporter/__init__.py | 340 ++++++++++++++++++ .../opentelemetry/exporter/otlp/version.py | 15 + .../tests/__init__.py | 0 .../tests/test_otlp_trace_exporter.py | 277 ++++++++++++++ 11 files changed, 1022 insertions(+) create mode 100644 exporter/opentelemetry-exporter-otlp/CHANGELOG.md create mode 100644 exporter/opentelemetry-exporter-otlp/LICENSE create mode 100644 exporter/opentelemetry-exporter-otlp/MANIFEST.in create mode 100644 exporter/opentelemetry-exporter-otlp/README.rst create mode 100644 exporter/opentelemetry-exporter-otlp/setup.cfg create mode 100644 exporter/opentelemetry-exporter-otlp/setup.py create mode 100644 exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py create mode 100644 exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py create mode 100644 exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py create mode 100644 exporter/opentelemetry-exporter-otlp/tests/__init__.py create mode 100644 exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md new file mode 100644 index 0000000000..bcaa7d1181 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +## Unreleased + +- Change package name to opentelemetry-exporter-otlp + ([#953](https://github.com/open-telemetry/opentelemetry-python/pull/953)) + +## Version 0.11b0 + +Released 2020-07-28 + +- Update span exporter to use OpenTelemetry Proto v0.4.0 ([#872](https://github.com/open-telemetry/opentelemetry-python/pull/889)) + +## 0.9b0 + +Released 2020-06-10 + +- Initial release diff --git a/exporter/opentelemetry-exporter-otlp/LICENSE b/exporter/opentelemetry-exporter-otlp/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/exporter/opentelemetry-exporter-otlp/MANIFEST.in b/exporter/opentelemetry-exporter-otlp/MANIFEST.in new file mode 100644 index 0000000000..aed3e33273 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/MANIFEST.in @@ -0,0 +1,9 @@ +graft src +graft tests +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__/* +include CHANGELOG.md +include MANIFEST.in +include README.rst +include LICENSE diff --git a/exporter/opentelemetry-exporter-otlp/README.rst b/exporter/opentelemetry-exporter-otlp/README.rst new file mode 100644 index 0000000000..8adf657181 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/README.rst @@ -0,0 +1,25 @@ +OpenTelemetry Collector Exporter +================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-exporter-otlp.svg + :target: https://pypi.org/project/opentelemetry-exporter-otlp/ + +This library allows to export data to the OpenTelemetry Collector using the OpenTelemetry Protocol. + +Installation +------------ + +:: + + pip install opentelemetry-exporter-otlp + + +References +---------- + +* `OpenTelemetry Collector Exporter `_ +* `OpenTelemetry Collector `_ +* `OpenTelemetry `_ +* `OpenTelemetry Protocol Specification `_ diff --git a/exporter/opentelemetry-exporter-otlp/setup.cfg b/exporter/opentelemetry-exporter-otlp/setup.cfg new file mode 100644 index 0000000000..262ac02008 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/setup.cfg @@ -0,0 +1,54 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-exporter-otlp +description = OpenTelemetry Collector Exporter +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/tree/master/exporter/opentelemetry-exporter-otlp +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + +[options] +python_requires = >=3.5 +package_dir= + =src +packages=find_namespace: +install_requires = + grpcio >= 1.0.0, < 2.0.0 + googleapis-common-protos ~= 1.52.0 + opentelemetry-api == 0.12.dev0 + opentelemetry-sdk == 0.12.dev0 + opentelemetry-proto == 0.12.dev0 + backoff ~= 1.10.0 + +[options.extras_require] +test = + pytest-grpc + +[options.packages.find] +where = src diff --git a/exporter/opentelemetry-exporter-otlp/setup.py b/exporter/opentelemetry-exporter-otlp/setup.py new file mode 100644 index 0000000000..c04c30fca4 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/setup.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "exporter", "otlp", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py new file mode 100644 index 0000000000..dca0042a68 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py @@ -0,0 +1,57 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +This library allows to export tracing data to an OTLP collector. + +Usage +----- + +The **OTLP Span Exporter** allows to export `OpenTelemetry`_ traces to the +`OTLP`_ collector. + + +.. _OTLP: https://github.com/open-telemetry/opentelemetry-collector/ +.. _OpenTelemetry: https://github.com/open-telemetry/opentelemetry-python/ + +.. code:: python + + from opentelemetry import trace + from opentelemetry.exporter.otlp.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + + # Resource can be required for some backends, e.g. Jaeger + # If resource wouldn't be set - traces wouldn't appears in Jaeger + resource = Resource(labels=labels={ + "service.name": "service" + }) + + trace.set_tracer_provider(TracerProvider(resource=resource))) + tracer = trace.get_tracer(__name__) + + otlp_exporter = OTLPSpanExporter(endpoint="localhost:55678") + + span_processor = BatchExportSpanProcessor(otlp_exporter) + + trace.get_tracer_provider().add_span_processor(span_processor) + + with tracer.start_as_current_span("foo"): + print("Hello world!") + +API +--- +""" diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py new file mode 100644 index 0000000000..47a862908e --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -0,0 +1,340 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OTLP Span Exporter""" + +import logging +from collections.abc import Mapping, Sequence +from time import sleep +from typing import Sequence as TypingSequence + +from backoff import expo +from google.rpc.error_details_pb2 import RetryInfo +from grpc import ( + ChannelCredentials, + RpcError, + StatusCode, + insecure_channel, + secure_channel, +) + +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import ( + TraceServiceStub, +) +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.resource.v1.resource_pb2 import Resource +from opentelemetry.proto.trace.v1.trace_pb2 import ( + InstrumentationLibrarySpans, + ResourceSpans, +) +from opentelemetry.proto.trace.v1.trace_pb2 import Span as CollectorSpan +from opentelemetry.proto.trace.v1.trace_pb2 import Status +from opentelemetry.sdk.trace import Span as SDKSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + +logger = logging.getLogger(__name__) + + +def _translate_key_values(key, value): + + if isinstance(value, bool): + any_value = AnyValue(bool_value=value) + + elif isinstance(value, str): + any_value = AnyValue(string_value=value) + + elif isinstance(value, int): + any_value = AnyValue(int_value=value) + + elif isinstance(value, float): + any_value = AnyValue(double_value=value) + + elif isinstance(value, Sequence): + any_value = AnyValue(array_value=value) + + elif isinstance(value, Mapping): + any_value = AnyValue(kvlist_value=value) + + else: + raise Exception( + "Invalid type {} of value {}".format(type(value), value) + ) + + return KeyValue(key=key, value=any_value) + + +# pylint: disable=no-member +class OTLPSpanExporter(SpanExporter): + """OTLP span exporter + + Args: + endpoint: OpenTelemetry Collector receiver endpoint + credentials: Credentials object for server authentication + metadata: Metadata to send when exporting + """ + + def __init__( + self, + endpoint="localhost:55678", + credentials: ChannelCredentials = None, + metadata=None, + ): + super().__init__() + + self._metadata = metadata + self._collector_span_kwargs = None + + if credentials is None: + self._client = TraceServiceStub(insecure_channel(endpoint)) + else: + self._client = TraceServiceStub( + secure_channel(endpoint, credentials) + ) + + def _translate_name(self, sdk_span): + self._collector_span_kwargs["name"] = sdk_span.name + + def _translate_start_time(self, sdk_span): + self._collector_span_kwargs[ + "start_time_unix_nano" + ] = sdk_span.start_time + + def _translate_end_time(self, sdk_span): + self._collector_span_kwargs["end_time_unix_nano"] = sdk_span.end_time + + def _translate_span_id(self, sdk_span): + self._collector_span_kwargs[ + "span_id" + ] = sdk_span.context.span_id.to_bytes(8, "big") + + def _translate_trace_id(self, sdk_span): + self._collector_span_kwargs[ + "trace_id" + ] = sdk_span.context.trace_id.to_bytes(16, "big") + + def _translate_parent(self, sdk_span): + if sdk_span.parent is not None: + self._collector_span_kwargs[ + "parent_span_id" + ] = sdk_span.parent.span_id.to_bytes(8, "big") + + def _translate_context_trace_state(self, sdk_span): + if sdk_span.context.trace_state is not None: + self._collector_span_kwargs["trace_state"] = ",".join( + [ + "{}={}".format(key, value) + for key, value in (sdk_span.context.trace_state.items()) + ] + ) + + def _translate_attributes(self, sdk_span): + if sdk_span.attributes: + + self._collector_span_kwargs["attributes"] = [] + + for key, value in sdk_span.attributes.items(): + + try: + self._collector_span_kwargs["attributes"].append( + _translate_key_values(key, value) + ) + except Exception as error: # pylint: disable=broad-except + logger.exception(error) + + def _translate_events(self, sdk_span): + if sdk_span.events: + self._collector_span_kwargs["events"] = [] + + for sdk_span_event in sdk_span.events: + + collector_span_event = CollectorSpan.Event( + name=sdk_span_event.name, + time_unix_nano=sdk_span_event.timestamp, + ) + + for key, value in sdk_span_event.attributes.items(): + try: + collector_span_event.attributes.append( + _translate_key_values(key, value) + ) + # pylint: disable=broad-except + except Exception as error: + logger.exception(error) + + self._collector_span_kwargs["events"].append( + collector_span_event + ) + + def _translate_links(self, sdk_span): + if sdk_span.links: + self._collector_span_kwargs["links"] = [] + + for sdk_span_link in sdk_span.links: + + collector_span_link = CollectorSpan.Link( + trace_id=( + sdk_span_link.context.trace_id.to_bytes(16, "big") + ), + span_id=(sdk_span_link.context.span_id.to_bytes(8, "big")), + ) + + for key, value in sdk_span_link.attributes.items(): + try: + collector_span_link.attributes.append( + _translate_key_values(key, value) + ) + # pylint: disable=broad-except + except Exception as error: + logger.exception(error) + + self._collector_span_kwargs["links"].append( + collector_span_link + ) + + def _translate_status(self, sdk_span): + if sdk_span.status is not None: + self._collector_span_kwargs["status"] = Status( + code=sdk_span.status.canonical_code.value, + message=sdk_span.status.description, + ) + + def _translate_spans( + self, sdk_spans: TypingSequence[SDKSpan], + ) -> ExportTraceServiceRequest: + + sdk_resource_instrumentation_library_spans = {} + + for sdk_span in sdk_spans: + + if sdk_span.resource not in ( + sdk_resource_instrumentation_library_spans.keys() + ): + sdk_resource_instrumentation_library_spans[ + sdk_span.resource + ] = InstrumentationLibrarySpans() + + self._collector_span_kwargs = {} + + self._translate_name(sdk_span) + self._translate_start_time(sdk_span) + self._translate_end_time(sdk_span) + self._translate_span_id(sdk_span) + self._translate_trace_id(sdk_span) + self._translate_parent(sdk_span) + self._translate_context_trace_state(sdk_span) + self._translate_attributes(sdk_span) + self._translate_events(sdk_span) + self._translate_links(sdk_span) + self._translate_status(sdk_span) + + self._collector_span_kwargs["kind"] = getattr( + CollectorSpan.SpanKind, sdk_span.kind.name + ) + + sdk_resource_instrumentation_library_spans[ + sdk_span.resource + ].spans.append(CollectorSpan(**self._collector_span_kwargs)) + + resource_spans = [] + + for ( + sdk_resource, + instrumentation_library_spans, + ) in sdk_resource_instrumentation_library_spans.items(): + + collector_resource = Resource() + + for key, value in sdk_resource.labels.items(): + + try: + collector_resource.attributes.append( + _translate_key_values(key, value) + ) + except Exception as error: # pylint: disable=broad-except + logger.exception(error) + + resource_spans.append( + ResourceSpans( + resource=collector_resource, + instrumentation_library_spans=[ + instrumentation_library_spans + ], + ) + ) + + return ExportTraceServiceRequest(resource_spans=resource_spans) + + def export(self, spans: TypingSequence[SDKSpan]) -> SpanExportResult: + # expo returns a generator that yields delay values which grow + # exponentially. Once delay is greater than max_value, the yielded + # value will remain constant. + # max_value is set to 900 (900 seconds is 15 minutes) to use the same + # value as used in the Go implementation. + + max_value = 900 + + for delay in expo(max_value=max_value): + + if delay == max_value: + return SpanExportResult.FAILURE + + try: + self._client.Export( + request=self._translate_spans(spans), + metadata=self._metadata, + ) + + return SpanExportResult.SUCCESS + + except RpcError as error: + + if error.code() in [ + StatusCode.CANCELLED, + StatusCode.DEADLINE_EXCEEDED, + StatusCode.PERMISSION_DENIED, + StatusCode.UNAUTHENTICATED, + StatusCode.RESOURCE_EXHAUSTED, + StatusCode.ABORTED, + StatusCode.OUT_OF_RANGE, + StatusCode.UNAVAILABLE, + StatusCode.DATA_LOSS, + ]: + + retry_info_bin = dict(error.trailing_metadata()).get( + "google.rpc.retryinfo-bin" + ) + if retry_info_bin is not None: + retry_info = RetryInfo() + retry_info.ParseFromString(retry_info_bin) + delay = ( + retry_info.retry_delay.seconds + + retry_info.retry_delay.nanos / 1.0e9 + ) + + logger.debug("Waiting %ss before retrying export of span") + sleep(delay) + continue + + if error.code() == StatusCode.OK: + return SpanExportResult.SUCCESS + + return SpanExportResult.FAILURE + + return SpanExportResult.FAILURE + + def shutdown(self): + pass diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py new file mode 100644 index 0000000000..780a92b6a1 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.12.dev0" diff --git a/exporter/opentelemetry-exporter-otlp/tests/__init__.py b/exporter/opentelemetry-exporter-otlp/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py new file mode 100644 index 0000000000..c7e26508b2 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -0,0 +1,277 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +from unittest import TestCase +from unittest.mock import Mock, PropertyMock, patch + +from google.protobuf.duration_pb2 import Duration +from google.rpc.error_details_pb2 import RetryInfo +from grpc import StatusCode, server + +from opentelemetry.exporter.otlp.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) +from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import ( + TraceServiceServicer, + add_TraceServiceServicer_to_server, +) +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.resource.v1.resource_pb2 import ( + Resource as CollectorResource, +) +from opentelemetry.proto.trace.v1.trace_pb2 import ( + InstrumentationLibrarySpans, + ResourceSpans, +) +from opentelemetry.proto.trace.v1.trace_pb2 import Span as CollectorSpan +from opentelemetry.proto.trace.v1.trace_pb2 import Status +from opentelemetry.sdk.resources import Resource as SDKResource +from opentelemetry.sdk.trace import Span, TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleExportSpanProcessor, + SpanExportResult, +) +from opentelemetry.trace import SpanKind + + +class TraceServiceServicerUNAVAILABLEDelay(TraceServiceServicer): + # pylint: disable=invalid-name,unused-argument,no-self-use + def Export(self, request, context): + context.set_code(StatusCode.UNAVAILABLE) + + context.send_initial_metadata( + (("google.rpc.retryinfo-bin", RetryInfo().SerializeToString()),) + ) + context.set_trailing_metadata( + ( + ( + "google.rpc.retryinfo-bin", + RetryInfo( + retry_delay=Duration(seconds=4) + ).SerializeToString(), + ), + ) + ) + + return ExportTraceServiceResponse() + + +class TraceServiceServicerUNAVAILABLE(TraceServiceServicer): + # pylint: disable=invalid-name,unused-argument,no-self-use + def Export(self, request, context): + context.set_code(StatusCode.UNAVAILABLE) + + return ExportTraceServiceResponse() + + +class TraceServiceServicerSUCCESS(TraceServiceServicer): + # pylint: disable=invalid-name,unused-argument,no-self-use + def Export(self, request, context): + context.set_code(StatusCode.OK) + + return ExportTraceServiceResponse() + + +class TestOTLPSpanExporter(TestCase): + def setUp(self): + tracer_provider = TracerProvider() + self.exporter = OTLPSpanExporter() + tracer_provider.add_span_processor( + SimpleExportSpanProcessor(self.exporter) + ) + self.tracer = tracer_provider.get_tracer(__name__) + + self.server = server(ThreadPoolExecutor(max_workers=10)) + + self.server.add_insecure_port("[::]:55678") + + self.server.start() + + event_mock = Mock( + **{ + "timestamp": 1591240820506462784, + "attributes": OrderedDict([("a", 1), ("b", False)]), + } + ) + + type(event_mock).name = PropertyMock(return_value="a") + + self.span = Span( + "a", + context=Mock( + **{ + "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "span_id": 10217189687419569865, + "trace_id": 67545097771067222548457157018666467027, + } + ), + resource=SDKResource(OrderedDict([("a", 1), ("b", False)])), + parent=Mock(**{"span_id": 12345}), + attributes=OrderedDict([("a", 1), ("b", True)]), + events=[event_mock], + links=[ + Mock( + **{ + "context.trace_id": 1, + "context.span_id": 2, + "attributes": OrderedDict([("a", 1), ("b", False)]), + "kind": SpanKind.INTERNAL, + } + ) + ], + ) + + self.span.start() + self.span.end() + + def tearDown(self): + self.server.stop(None) + + @patch("opentelemetry.exporter.otlp.trace_exporter.expo") + @patch("opentelemetry.exporter.otlp.trace_exporter.sleep") + def test_unavailable(self, mock_sleep, mock_expo): + + mock_expo.configure_mock(**{"return_value": [1]}) + + add_TraceServiceServicer_to_server( + TraceServiceServicerUNAVAILABLE(), self.server + ) + self.assertEqual( + self.exporter.export([self.span]), SpanExportResult.FAILURE + ) + mock_sleep.assert_called_with(1) + + @patch("opentelemetry.exporter.otlp.trace_exporter.expo") + @patch("opentelemetry.exporter.otlp.trace_exporter.sleep") + def test_unavailable_delay(self, mock_sleep, mock_expo): + + mock_expo.configure_mock(**{"return_value": [1]}) + + add_TraceServiceServicer_to_server( + TraceServiceServicerUNAVAILABLEDelay(), self.server + ) + self.assertEqual( + self.exporter.export([self.span]), SpanExportResult.FAILURE + ) + mock_sleep.assert_called_with(4) + + def test_success(self): + add_TraceServiceServicer_to_server( + TraceServiceServicerSUCCESS(), self.server + ) + self.assertEqual( + self.exporter.export([self.span]), SpanExportResult.SUCCESS + ) + + def test_translate_spans(self): + + expected = ExportTraceServiceRequest( + resource_spans=[ + ResourceSpans( + resource=CollectorResource( + attributes=[ + KeyValue(key="a", value=AnyValue(int_value=1)), + KeyValue( + key="b", value=AnyValue(bool_value=False) + ), + ] + ), + instrumentation_library_spans=[ + InstrumentationLibrarySpans( + spans=[ + CollectorSpan( + # pylint: disable=no-member + name="a", + start_time_unix_nano=self.span.start_time, + end_time_unix_nano=self.span.end_time, + trace_state="a=b,c=d", + span_id=int.to_bytes( + 10217189687419569865, 8, "big" + ), + trace_id=int.to_bytes( + 67545097771067222548457157018666467027, + 16, + "big", + ), + parent_span_id=( + b"\000\000\000\000\000\00009" + ), + kind=CollectorSpan.SpanKind.INTERNAL, + attributes=[ + KeyValue( + key="a", + value=AnyValue(int_value=1), + ), + KeyValue( + key="b", + value=AnyValue(bool_value=True), + ), + ], + events=[ + CollectorSpan.Event( + name="a", + time_unix_nano=1591240820506462784, + attributes=[ + KeyValue( + key="a", + value=AnyValue( + int_value=1 + ), + ), + KeyValue( + key="b", + value=AnyValue( + bool_value=False + ), + ), + ], + ) + ], + status=Status(code=0, message=""), + links=[ + CollectorSpan.Link( + trace_id=int.to_bytes( + 1, 16, "big" + ), + span_id=int.to_bytes(2, 8, "big"), + attributes=[ + KeyValue( + key="a", + value=AnyValue( + int_value=1 + ), + ), + KeyValue( + key="b", + value=AnyValue( + bool_value=False + ), + ), + ], + ) + ], + ) + ] + ) + ], + ), + ] + ) + + # pylint: disable=protected-access + self.assertEqual(expected, self.exporter._translate_spans([self.span])) From 01daee4bcefdaa9938af8adb82bed31e06f54053 Mon Sep 17 00:00:00 2001 From: alrex Date: Fri, 7 Aug 2020 14:57:04 -0700 Subject: [PATCH 02/15] fix: update default OTLP port to 55680 (#977) --- exporter/opentelemetry-exporter-otlp/CHANGELOG.md | 2 ++ .../src/opentelemetry/exporter/otlp/__init__.py | 2 +- .../src/opentelemetry/exporter/otlp/trace_exporter/__init__.py | 2 +- .../tests/test_otlp_trace_exporter.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index bcaa7d1181..2379a94cb3 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -4,6 +4,8 @@ - Change package name to opentelemetry-exporter-otlp ([#953](https://github.com/open-telemetry/opentelemetry-python/pull/953)) +- Update default port to 55680 + ([#977](https://github.com/open-telemetry/opentelemetry-python/pull/977)) ## Version 0.11b0 diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py index dca0042a68..a078cb7ccc 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py @@ -43,7 +43,7 @@ trace.set_tracer_provider(TracerProvider(resource=resource))) tracer = trace.get_tracer(__name__) - otlp_exporter = OTLPSpanExporter(endpoint="localhost:55678") + otlp_exporter = OTLPSpanExporter(endpoint="localhost:55680") span_processor = BatchExportSpanProcessor(otlp_exporter) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py index 47a862908e..be1419fa39 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -89,7 +89,7 @@ class OTLPSpanExporter(SpanExporter): def __init__( self, - endpoint="localhost:55678", + endpoint="localhost:55680", credentials: ChannelCredentials = None, metadata=None, ): diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index c7e26508b2..9058937f87 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -98,7 +98,7 @@ def setUp(self): self.server = server(ThreadPoolExecutor(max_workers=10)) - self.server.add_insecure_port("[::]:55678") + self.server.add_insecure_port("[::]:55680") self.server.start() From 09c3828e660fe68b475998692e09596362df796b Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Sat, 15 Aug 2020 18:06:27 -0700 Subject: [PATCH 03/15] chore: 0.13.dev0 version update (#991) --- exporter/opentelemetry-exporter-otlp/CHANGELOG.md | 4 ++++ exporter/opentelemetry-exporter-otlp/setup.cfg | 6 +++--- .../src/opentelemetry/exporter/otlp/version.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 2379a94cb3..24eb8bcf58 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## Version 0.12b0 + +Released 2020-08-14 + - Change package name to opentelemetry-exporter-otlp ([#953](https://github.com/open-telemetry/opentelemetry-python/pull/953)) - Update default port to 55680 diff --git a/exporter/opentelemetry-exporter-otlp/setup.cfg b/exporter/opentelemetry-exporter-otlp/setup.cfg index 262ac02008..3a6ca8f957 100644 --- a/exporter/opentelemetry-exporter-otlp/setup.cfg +++ b/exporter/opentelemetry-exporter-otlp/setup.cfg @@ -41,9 +41,9 @@ packages=find_namespace: install_requires = grpcio >= 1.0.0, < 2.0.0 googleapis-common-protos ~= 1.52.0 - opentelemetry-api == 0.12.dev0 - opentelemetry-sdk == 0.12.dev0 - opentelemetry-proto == 0.12.dev0 + opentelemetry-api == 0.13dev0 + opentelemetry-sdk == 0.13dev0 + opentelemetry-proto == 0.13dev0 backoff ~= 1.10.0 [options.extras_require] diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py index 780a92b6a1..9cc445d09e 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.12.dev0" +__version__ = "0.13dev0" From c40f200e667bb10ebfffe83e1fb5412e0406c441 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Mon, 17 Aug 2020 22:06:05 -0600 Subject: [PATCH 04/15] exporter/otlp: Add OTLP metric exporter (#835) --- .../opentelemetry-exporter-otlp/CHANGELOG.md | 5 +- .../opentelemetry/exporter/otlp/exporter.py | 194 +++++++++++++++++ .../otlp/metrics_exporter/__init__.py | 198 ++++++++++++++++++ .../exporter/otlp/trace_exporter/__init__.py | 172 ++------------- .../tests/test_otlp_metric_exporter.py | 116 ++++++++++ .../tests/test_otlp_trace_exporter.py | 10 +- 6 files changed, 535 insertions(+), 160 deletions(-) create mode 100644 exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py create mode 100644 exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py create mode 100644 exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 24eb8bcf58..ed6aee3b4c 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,9 +2,10 @@ ## Unreleased -## Version 0.12b0 +- Add metric OTLP exporter + ([#835](https://github.com/open-telemetry/opentelemetry-python/pull/835)) -Released 2020-08-14 +## Version 0.12b0 - Change package name to opentelemetry-exporter-otlp ([#953](https://github.com/open-telemetry/opentelemetry-python/pull/953)) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py new file mode 100644 index 0000000000..0ce7ef6617 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py @@ -0,0 +1,194 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OTLP Exporter""" + +import logging +from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence +from time import sleep + +from backoff import expo +from google.rpc.error_details_pb2 import RetryInfo +from grpc import ( + ChannelCredentials, + RpcError, + StatusCode, + insecure_channel, + secure_channel, +) + +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.resource.v1.resource_pb2 import Resource + +logger = logging.getLogger(__name__) + + +def _translate_key_values(key, value): + + if isinstance(value, bool): + any_value = AnyValue(bool_value=value) + + elif isinstance(value, str): + any_value = AnyValue(string_value=value) + + elif isinstance(value, int): + any_value = AnyValue(int_value=value) + + elif isinstance(value, float): + any_value = AnyValue(double_value=value) + + elif isinstance(value, Sequence): + any_value = AnyValue(array_value=value) + + elif isinstance(value, Mapping): + any_value = AnyValue(kvlist_value=value) + + else: + raise Exception( + "Invalid type {} of value {}".format(type(value), value) + ) + + return KeyValue(key=key, value=any_value) + + +def _get_resource_data( + sdk_resource_instrumentation_library_data, resource_class, name +): + + resource_data = [] + + for ( + sdk_resource, + instrumentation_library_data, + ) in sdk_resource_instrumentation_library_data.items(): + + collector_resource = Resource() + + for key, value in sdk_resource.labels.items(): + + try: + # pylint: disable=no-member + collector_resource.attributes.append( + _translate_key_values(key, value) + ) + except Exception as error: # pylint: disable=broad-except + logger.exception(error) + + resource_data.append( + resource_class( + **{ + "resource": collector_resource, + "instrumentation_library_{}".format(name): [ + instrumentation_library_data + ], + } + ) + ) + + return resource_data + + +# pylint: disable=no-member +class OTLPExporterMixin(ABC): + """OTLP span/metric exporter + + Args: + endpoint: OpenTelemetry Collector receiver endpoint + credentials: ChannelCredentials object for server authentication + metadata: Metadata to send when exporting + """ + + def __init__( + self, + endpoint: str = "localhost:55680", + credentials: ChannelCredentials = None, + metadata: tuple = None, + ): + super().__init__() + + self._metadata = metadata + self._collector_span_kwargs = None + + if credentials is None: + self._client = self._stub(insecure_channel(endpoint)) + else: + self._client = self._stub(secure_channel(endpoint, credentials)) + + @abstractmethod + def _translate_data(self, data): + pass + + def _export(self, data): + # expo returns a generator that yields delay values which grow + # exponentially. Once delay is greater than max_value, the yielded + # value will remain constant. + # max_value is set to 900 (900 seconds is 15 minutes) to use the same + # value as used in the Go implementation. + + max_value = 900 + + for delay in expo(max_value=max_value): + + if delay == max_value: + return self._result.FAILURE + + try: + self._client.Export( + request=self._translate_data(data), + metadata=self._metadata, + ) + + return self._result.SUCCESS + + except RpcError as error: + + if error.code() in [ + StatusCode.CANCELLED, + StatusCode.DEADLINE_EXCEEDED, + StatusCode.PERMISSION_DENIED, + StatusCode.UNAUTHENTICATED, + StatusCode.RESOURCE_EXHAUSTED, + StatusCode.ABORTED, + StatusCode.OUT_OF_RANGE, + StatusCode.UNAVAILABLE, + StatusCode.DATA_LOSS, + ]: + + retry_info_bin = dict(error.trailing_metadata()).get( + "google.rpc.retryinfo-bin" + ) + if retry_info_bin is not None: + retry_info = RetryInfo() + retry_info.ParseFromString(retry_info_bin) + delay = ( + retry_info.retry_delay.seconds + + retry_info.retry_delay.nanos / 1.0e9 + ) + + logger.debug( + "Waiting %ss before retrying export of span", delay + ) + sleep(delay) + continue + + if error.code() == StatusCode.OK: + return self._result.SUCCESS + + return self.result.FAILURE + + return self._result.FAILURE + + def shutdown(self): + pass diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py new file mode 100644 index 0000000000..944428e37d --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py @@ -0,0 +1,198 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OTLP Metrics Exporter""" + +import logging +from typing import Sequence + +# pylint: disable=duplicate-code +from opentelemetry.exporter.otlp.exporter import ( + OTLPExporterMixin, + _get_resource_data, +) +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( + ExportMetricsServiceRequest, +) +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2_grpc import ( + MetricsServiceStub, +) +from opentelemetry.proto.common.v1.common_pb2 import StringKeyValue +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + DoubleDataPoint, + InstrumentationLibraryMetrics, + Int64DataPoint, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + Metric as CollectorMetric, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + MetricDescriptor, + ResourceMetrics, +) +from opentelemetry.sdk.metrics import Counter +from opentelemetry.sdk.metrics import Metric as SDKMetric +from opentelemetry.sdk.metrics import ( + SumObserver, + UpDownCounter, + UpDownSumObserver, + ValueObserver, + ValueRecorder, +) +from opentelemetry.sdk.metrics.export import ( + MetricsExporter, + MetricsExportResult, +) + +logger = logging.getLogger(__name__) + + +def _get_data_points(sdk_metric, data_point_class): + + data_points = [] + + for ( + label, + bound_counter, + ) in sdk_metric.instrument.bound_instruments.items(): + + string_key_values = [] + + for label_key, label_value in label: + string_key_values.append( + StringKeyValue(key=label_key, value=label_value) + ) + + for view_data in bound_counter.view_datas: + + if view_data.labels == label: + + data_points.append( + data_point_class( + labels=string_key_values, + value=view_data.aggregator.current, + ) + ) + break + + return data_points + + +def _get_temporality(instrument): + # pylint: disable=no-member + if isinstance(instrument, (Counter, UpDownCounter)): + temporality = MetricDescriptor.Temporality.DELTA + elif isinstance(instrument, (ValueRecorder, ValueObserver)): + temporality = MetricDescriptor.Temporality.INSTANTANEOUS + elif isinstance(instrument, (SumObserver, UpDownSumObserver)): + temporality = MetricDescriptor.Temporality.CUMULATIVE + else: + raise Exception( + "No temporality defined for instrument type {}".format( + type(instrument) + ) + ) + + return temporality + + +def _get_type(value_type): + # pylint: disable=no-member + if value_type is int: + type_ = MetricDescriptor.Type.INT64 + + elif value_type is float: + type_ = MetricDescriptor.Type.DOUBLE + + # FIXME What are the types that correspond with + # MetricDescriptor.Type.HISTOGRAM and + # MetricDescriptor.Type.SUMMARY? + else: + raise Exception( + "No type defined for valie type {}".format(type(value_type)) + ) + + return type_ + + +class OTLPMetricsExporter(MetricsExporter, OTLPExporterMixin): + """OTLP metrics exporter + + Args: + endpoint: OpenTelemetry Collector receiver endpoint + credentials: Credentials object for server authentication + metadata: Metadata to send when exporting + """ + + _stub = MetricsServiceStub + _result = MetricsExportResult + + def _translate_data(self, data): + # pylint: disable=too-many-locals,no-member + # pylint: disable=attribute-defined-outside-init + + sdk_resource_instrumentation_library_metrics = {} + + for sdk_metric in data: + + if sdk_metric.instrument.meter.resource not in ( + sdk_resource_instrumentation_library_metrics.keys() + ): + sdk_resource_instrumentation_library_metrics[ + sdk_metric.instrument.meter.resource + ] = InstrumentationLibraryMetrics() + + self._metric_descriptor_kwargs = {} + + metric_descriptor = MetricDescriptor( + name=sdk_metric.instrument.name, + description=sdk_metric.instrument.description, + unit=sdk_metric.instrument.unit, + type=_get_type(sdk_metric.instrument.value_type), + temporality=_get_temporality(sdk_metric.instrument), + ) + + if metric_descriptor.type == MetricDescriptor.Type.INT64: + + collector_metric = CollectorMetric( + metric_descriptor=metric_descriptor, + int64_data_points=_get_data_points( + sdk_metric, Int64DataPoint + ), + ) + + elif metric_descriptor.type == MetricDescriptor.Type.DOUBLE: + + collector_metric = CollectorMetric( + metric_descriptor=metric_descriptor, + double_data_points=_get_data_points( + sdk_metric, DoubleDataPoint + ), + ) + + sdk_resource_instrumentation_library_metrics[ + sdk_metric.instrument.meter.resource + ].metrics.append(collector_metric) + + return ExportMetricsServiceRequest( + resource_metrics=_get_resource_data( + sdk_resource_instrumentation_library_metrics, + ResourceMetrics, + "metrics", + ) + ) + + def export(self, metrics: Sequence[SDKMetric]) -> MetricsExportResult: + # pylint: disable=arguments-differ + return self._export(metrics) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py index be1419fa39..5a9a74a304 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -1,5 +1,4 @@ # Copyright The OpenTelemetry Authors -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -15,28 +14,19 @@ """OTLP Span Exporter""" import logging -from collections.abc import Mapping, Sequence -from time import sleep -from typing import Sequence as TypingSequence - -from backoff import expo -from google.rpc.error_details_pb2 import RetryInfo -from grpc import ( - ChannelCredentials, - RpcError, - StatusCode, - insecure_channel, - secure_channel, -) +from typing import Sequence +from opentelemetry.exporter.otlp.exporter import ( + OTLPExporterMixin, + _get_resource_data, + _translate_key_values, +) from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( ExportTraceServiceRequest, ) from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import ( TraceServiceStub, ) -from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue -from opentelemetry.proto.resource.v1.resource_pb2 import Resource from opentelemetry.proto.trace.v1.trace_pb2 import ( InstrumentationLibrarySpans, ResourceSpans, @@ -49,36 +39,8 @@ logger = logging.getLogger(__name__) -def _translate_key_values(key, value): - - if isinstance(value, bool): - any_value = AnyValue(bool_value=value) - - elif isinstance(value, str): - any_value = AnyValue(string_value=value) - - elif isinstance(value, int): - any_value = AnyValue(int_value=value) - - elif isinstance(value, float): - any_value = AnyValue(double_value=value) - - elif isinstance(value, Sequence): - any_value = AnyValue(array_value=value) - - elif isinstance(value, Mapping): - any_value = AnyValue(kvlist_value=value) - - else: - raise Exception( - "Invalid type {} of value {}".format(type(value), value) - ) - - return KeyValue(key=key, value=any_value) - - # pylint: disable=no-member -class OTLPSpanExporter(SpanExporter): +class OTLPSpanExporter(SpanExporter, OTLPExporterMixin): """OTLP span exporter Args: @@ -87,23 +49,8 @@ class OTLPSpanExporter(SpanExporter): metadata: Metadata to send when exporting """ - def __init__( - self, - endpoint="localhost:55680", - credentials: ChannelCredentials = None, - metadata=None, - ): - super().__init__() - - self._metadata = metadata - self._collector_span_kwargs = None - - if credentials is None: - self._client = TraceServiceStub(insecure_channel(endpoint)) - else: - self._client = TraceServiceStub( - secure_channel(endpoint, credentials) - ) + _result = SpanExportResult + _stub = TraceServiceStub def _translate_name(self, sdk_span): self._collector_span_kwargs["name"] = sdk_span.name @@ -212,13 +159,11 @@ def _translate_status(self, sdk_span): message=sdk_span.status.description, ) - def _translate_spans( - self, sdk_spans: TypingSequence[SDKSpan], - ) -> ExportTraceServiceRequest: + def _translate_data(self, data) -> ExportTraceServiceRequest: sdk_resource_instrumentation_library_spans = {} - for sdk_span in sdk_spans: + for sdk_span in data: if sdk_span.resource not in ( sdk_resource_instrumentation_library_spans.keys() @@ -249,92 +194,13 @@ def _translate_spans( sdk_span.resource ].spans.append(CollectorSpan(**self._collector_span_kwargs)) - resource_spans = [] - - for ( - sdk_resource, - instrumentation_library_spans, - ) in sdk_resource_instrumentation_library_spans.items(): - - collector_resource = Resource() - - for key, value in sdk_resource.labels.items(): - - try: - collector_resource.attributes.append( - _translate_key_values(key, value) - ) - except Exception as error: # pylint: disable=broad-except - logger.exception(error) - - resource_spans.append( - ResourceSpans( - resource=collector_resource, - instrumentation_library_spans=[ - instrumentation_library_spans - ], - ) + return ExportTraceServiceRequest( + resource_spans=_get_resource_data( + sdk_resource_instrumentation_library_spans, + ResourceSpans, + "spans", ) + ) - return ExportTraceServiceRequest(resource_spans=resource_spans) - - def export(self, spans: TypingSequence[SDKSpan]) -> SpanExportResult: - # expo returns a generator that yields delay values which grow - # exponentially. Once delay is greater than max_value, the yielded - # value will remain constant. - # max_value is set to 900 (900 seconds is 15 minutes) to use the same - # value as used in the Go implementation. - - max_value = 900 - - for delay in expo(max_value=max_value): - - if delay == max_value: - return SpanExportResult.FAILURE - - try: - self._client.Export( - request=self._translate_spans(spans), - metadata=self._metadata, - ) - - return SpanExportResult.SUCCESS - - except RpcError as error: - - if error.code() in [ - StatusCode.CANCELLED, - StatusCode.DEADLINE_EXCEEDED, - StatusCode.PERMISSION_DENIED, - StatusCode.UNAUTHENTICATED, - StatusCode.RESOURCE_EXHAUSTED, - StatusCode.ABORTED, - StatusCode.OUT_OF_RANGE, - StatusCode.UNAVAILABLE, - StatusCode.DATA_LOSS, - ]: - - retry_info_bin = dict(error.trailing_metadata()).get( - "google.rpc.retryinfo-bin" - ) - if retry_info_bin is not None: - retry_info = RetryInfo() - retry_info.ParseFromString(retry_info_bin) - delay = ( - retry_info.retry_delay.seconds - + retry_info.retry_delay.nanos / 1.0e9 - ) - - logger.debug("Waiting %ss before retrying export of span") - sleep(delay) - continue - - if error.code() == StatusCode.OK: - return SpanExportResult.SUCCESS - - return SpanExportResult.FAILURE - - return SpanExportResult.FAILURE - - def shutdown(self): - pass + def export(self, spans: Sequence[SDKSpan]) -> SpanExportResult: + return self._export(spans) diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py new file mode 100644 index 0000000000..20fecd44a2 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py @@ -0,0 +1,116 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +from unittest import TestCase + +from opentelemetry.exporter.otlp.metrics_exporter import OTLPMetricsExporter +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( + ExportMetricsServiceRequest, +) +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue, + KeyValue, + StringKeyValue, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + InstrumentationLibraryMetrics, + Int64DataPoint, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + Metric as CollectorMetric, +) +from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + MetricDescriptor, + ResourceMetrics, +) +from opentelemetry.proto.resource.v1.resource_pb2 import ( + Resource as CollectorResource, +) +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export import MetricRecord +from opentelemetry.sdk.metrics.export.aggregate import SumAggregator +from opentelemetry.sdk.resources import Resource as SDKResource + + +class TestOTLPMetricExporter(TestCase): + def setUp(self): + self.exporter = OTLPMetricsExporter() + + self.counter_metric_record = MetricRecord( + Counter( + "a", + "b", + "c", + int, + MeterProvider( + resource=SDKResource(OrderedDict([("a", 1), ("b", False)])) + ).get_meter(__name__), + ("d",), + ), + OrderedDict([("e", "f")]), + SumAggregator(), + ) + + def test_translate_metrics(self): + # pylint: disable=no-member + + self.counter_metric_record.instrument.add(1, OrderedDict([("a", "b")])) + + expected = ExportMetricsServiceRequest( + resource_metrics=[ + ResourceMetrics( + resource=CollectorResource( + attributes=[ + KeyValue(key="a", value=AnyValue(int_value=1)), + KeyValue( + key="b", value=AnyValue(bool_value=False) + ), + ] + ), + instrumentation_library_metrics=[ + InstrumentationLibraryMetrics( + metrics=[ + CollectorMetric( + metric_descriptor=MetricDescriptor( + name="a", + description="b", + unit="c", + type=MetricDescriptor.Type.INT64, + temporality=( + MetricDescriptor.Temporality.DELTA + ), + ), + int64_data_points=[ + Int64DataPoint( + labels=[ + StringKeyValue( + key="a", value="b" + ) + ], + value=1, + ) + ], + ) + ] + ) + ], + ) + ] + ) + + # pylint: disable=protected-access + actual = self.exporter._translate_data([self.counter_metric_record]) + + self.assertEqual(expected, actual) diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index 9058937f87..b0ec8e4517 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -142,8 +142,8 @@ def setUp(self): def tearDown(self): self.server.stop(None) - @patch("opentelemetry.exporter.otlp.trace_exporter.expo") - @patch("opentelemetry.exporter.otlp.trace_exporter.sleep") + @patch("opentelemetry.exporter.otlp.exporter.expo") + @patch("opentelemetry.exporter.otlp.exporter.sleep") def test_unavailable(self, mock_sleep, mock_expo): mock_expo.configure_mock(**{"return_value": [1]}) @@ -156,8 +156,8 @@ def test_unavailable(self, mock_sleep, mock_expo): ) mock_sleep.assert_called_with(1) - @patch("opentelemetry.exporter.otlp.trace_exporter.expo") - @patch("opentelemetry.exporter.otlp.trace_exporter.sleep") + @patch("opentelemetry.exporter.otlp.exporter.expo") + @patch("opentelemetry.exporter.otlp.exporter.sleep") def test_unavailable_delay(self, mock_sleep, mock_expo): mock_expo.configure_mock(**{"return_value": [1]}) @@ -274,4 +274,4 @@ def test_translate_spans(self): ) # pylint: disable=protected-access - self.assertEqual(expected, self.exporter._translate_spans([self.span])) + self.assertEqual(expected, self.exporter._translate_data([self.span])) From f504c0bdd4593ae47a1fd063d9e7057d804c57e1 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Thu, 3 Sep 2020 17:44:01 -0600 Subject: [PATCH 05/15] Add missing underscore (#1073) Fixes #1072 --- .../src/opentelemetry/exporter/otlp/exporter.py | 2 +- .../tests/test_otlp_trace_exporter.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py index 0ce7ef6617..e0914f262f 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py @@ -186,7 +186,7 @@ def _export(self, data): if error.code() == StatusCode.OK: return self._result.SUCCESS - return self.result.FAILURE + return self._result.FAILURE return self._result.FAILURE diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index b0ec8e4517..a7f572323e 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -87,6 +87,14 @@ def Export(self, request, context): return ExportTraceServiceResponse() +class TraceServiceServicerALREADY_EXISTS(TraceServiceServicer): + # pylint: disable=invalid-name,unused-argument,no-self-use + def Export(self, request, context): + context.set_code(StatusCode.ALREADY_EXISTS) + + return ExportTraceServiceResponse() + + class TestOTLPSpanExporter(TestCase): def setUp(self): tracer_provider = TracerProvider() @@ -178,6 +186,14 @@ def test_success(self): self.exporter.export([self.span]), SpanExportResult.SUCCESS ) + def test_failure(self): + add_TraceServiceServicer_to_server( + TraceServiceServicerALREADY_EXISTS(), self.server + ) + self.assertEqual( + self.exporter.export([self.span]), SpanExportResult.FAILURE + ) + def test_translate_spans(self): expected = ExportTraceServiceRequest( From 1d8f5b8c41b921d0719c3accfe5dc6869e7f4793 Mon Sep 17 00:00:00 2001 From: alrex Date: Wed, 9 Sep 2020 08:23:18 -0700 Subject: [PATCH 06/15] sdk: rename resource labels to attributes (#1082) This aligns with the specification for Resources --- .../src/opentelemetry/exporter/otlp/__init__.py | 2 +- .../src/opentelemetry/exporter/otlp/exporter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py index a078cb7ccc..a4d8f46d4c 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/__init__.py @@ -36,7 +36,7 @@ # Resource can be required for some backends, e.g. Jaeger # If resource wouldn't be set - traces wouldn't appears in Jaeger - resource = Resource(labels=labels={ + resource = Resource(attributes={ "service.name": "service" }) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py index e0914f262f..7cd9f905e0 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py @@ -76,7 +76,7 @@ def _get_resource_data( collector_resource = Resource() - for key, value in sdk_resource.labels.items(): + for key, value in sdk_resource.attributes.items(): try: # pylint: disable=no-member From e86e4c844419427faf7cc3633f7b1b89113d904f Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Mon, 14 Sep 2020 17:15:03 -0600 Subject: [PATCH 07/15] exporter/otlp: Add instrumentation info to exported spans (#1095) Fixes #1094 --- .../opentelemetry-exporter-otlp/CHANGELOG.md | 2 ++ .../exporter/otlp/trace_exporter/__init__.py | 16 +++++++++++++++- .../tests/test_otlp_trace_exporter.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index ed6aee3b4c..49e6d8c941 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add instrumentation info to exported spans + ([#1095](https://github.com/open-telemetry/opentelemetry-python/pull/1095)) - Add metric OTLP exporter ([#835](https://github.com/open-telemetry/opentelemetry-python/pull/835)) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py index 5a9a74a304..fd1d8e235e 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -27,6 +27,7 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import ( TraceServiceStub, ) +from opentelemetry.proto.common.v1.common_pb2 import InstrumentationLibrary from opentelemetry.proto.trace.v1.trace_pb2 import ( InstrumentationLibrarySpans, ResourceSpans, @@ -168,9 +169,22 @@ def _translate_data(self, data) -> ExportTraceServiceRequest: if sdk_span.resource not in ( sdk_resource_instrumentation_library_spans.keys() ): + if sdk_span.instrumentation_info is not None: + instrumentation_library_spans = InstrumentationLibrarySpans( + instrumentation_library=InstrumentationLibrary( + name=sdk_span.instrumentation_info.name, + version=sdk_span.instrumentation_info.version, + ) + ) + + else: + instrumentation_library_spans = ( + InstrumentationLibrarySpans() + ) + sdk_resource_instrumentation_library_spans[ sdk_span.resource - ] = InstrumentationLibrarySpans() + ] = instrumentation_library_spans self._collector_span_kwargs = {} diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index a7f572323e..06a3877b92 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -30,7 +30,11 @@ TraceServiceServicer, add_TraceServiceServicer_to_server, ) -from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.common.v1.common_pb2 import ( + AnyValue, + InstrumentationLibrary, + KeyValue, +) from opentelemetry.proto.resource.v1.resource_pb2 import ( Resource as CollectorResource, ) @@ -46,6 +50,7 @@ SimpleExportSpanProcessor, SpanExportResult, ) +from opentelemetry.sdk.util.instrumentation import InstrumentationInfo from opentelemetry.trace import SpanKind @@ -142,6 +147,9 @@ def setUp(self): } ) ], + instrumentation_info=InstrumentationInfo( + name="name", version="version" + ), ) self.span.start() @@ -209,6 +217,9 @@ def test_translate_spans(self): ), instrumentation_library_spans=[ InstrumentationLibrarySpans( + instrumentation_library=InstrumentationLibrary( + name="name", version="version" + ), spans=[ CollectorSpan( # pylint: disable=no-member @@ -282,7 +293,7 @@ def test_translate_spans(self): ) ], ) - ] + ], ) ], ), From 4e2264ca499fb4d7913bb461455bcb10abddd7aa Mon Sep 17 00:00:00 2001 From: alrex Date: Thu, 17 Sep 2020 08:23:52 -0700 Subject: [PATCH 08/15] release: updating changelogs and version to 0.13b0 (#1129) * updating changelogs and version to 0.13b0 --- exporter/opentelemetry-exporter-otlp/CHANGELOG.md | 4 ++++ exporter/opentelemetry-exporter-otlp/setup.cfg | 6 +++--- .../src/opentelemetry/exporter/otlp/version.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 49e6d8c941..94ae999eca 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## Version 0.13b0 + +Released 2020-09-17 + - Add instrumentation info to exported spans ([#1095](https://github.com/open-telemetry/opentelemetry-python/pull/1095)) - Add metric OTLP exporter diff --git a/exporter/opentelemetry-exporter-otlp/setup.cfg b/exporter/opentelemetry-exporter-otlp/setup.cfg index 3a6ca8f957..af141f1b6b 100644 --- a/exporter/opentelemetry-exporter-otlp/setup.cfg +++ b/exporter/opentelemetry-exporter-otlp/setup.cfg @@ -41,9 +41,9 @@ packages=find_namespace: install_requires = grpcio >= 1.0.0, < 2.0.0 googleapis-common-protos ~= 1.52.0 - opentelemetry-api == 0.13dev0 - opentelemetry-sdk == 0.13dev0 - opentelemetry-proto == 0.13dev0 + opentelemetry-api == 0.13b0 + opentelemetry-sdk == 0.13b0 + opentelemetry-proto == 0.13b0 backoff ~= 1.10.0 [options.extras_require] diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py index 9cc445d09e..2015e87c70 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.13dev0" +__version__ = "0.13b0" From 35cb9fe3d242672052723f48c6fa74a5489d04ec Mon Sep 17 00:00:00 2001 From: alrex Date: Thu, 17 Sep 2020 12:21:39 -0700 Subject: [PATCH 09/15] chore: bump dev version (#1131) --- exporter/opentelemetry-exporter-otlp/setup.cfg | 6 +++--- .../src/opentelemetry/exporter/otlp/version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/setup.cfg b/exporter/opentelemetry-exporter-otlp/setup.cfg index af141f1b6b..12fdd62775 100644 --- a/exporter/opentelemetry-exporter-otlp/setup.cfg +++ b/exporter/opentelemetry-exporter-otlp/setup.cfg @@ -41,9 +41,9 @@ packages=find_namespace: install_requires = grpcio >= 1.0.0, < 2.0.0 googleapis-common-protos ~= 1.52.0 - opentelemetry-api == 0.13b0 - opentelemetry-sdk == 0.13b0 - opentelemetry-proto == 0.13b0 + opentelemetry-api == 0.14.dev0 + opentelemetry-sdk == 0.14.dev0 + opentelemetry-proto == 0.14.dev0 backoff ~= 1.10.0 [options.extras_require] diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py index 2015e87c70..0f99027898 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.13b0" +__version__ = "0.14.dev0" From 8bf1a2021388a86ddd50fafca1180d2e28a0aa39 Mon Sep 17 00:00:00 2001 From: Amos Law Date: Thu, 17 Sep 2020 12:43:36 -0700 Subject: [PATCH 10/15] Add type hints to OTLP exporter (#1121) --- .../opentelemetry-exporter-otlp/CHANGELOG.md | 2 ++ .../opentelemetry/exporter/otlp/exporter.py | 33 ++++++++++++----- .../otlp/metrics_exporter/__init__.py | 36 +++++++++++++------ .../exporter/otlp/trace_exporter/__init__.py | 33 ++++++++++------- 4 files changed, 72 insertions(+), 32 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 94ae999eca..a3686e1a10 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -10,6 +10,8 @@ Released 2020-09-17 ([#1095](https://github.com/open-telemetry/opentelemetry-python/pull/1095)) - Add metric OTLP exporter ([#835](https://github.com/open-telemetry/opentelemetry-python/pull/835)) +- Add type hints to OTLP exporter + ([#1121](https://github.com/open-telemetry/opentelemetry-python/pull/1121)) ## Version 0.12b0 diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py index 7cd9f905e0..079557f831 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/exporter.py @@ -18,6 +18,9 @@ from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from time import sleep +from typing import Any, Callable, Dict, Generic, List, Optional +from typing import Sequence as TypingSequence +from typing import Text, Tuple, TypeVar from backoff import expo from google.rpc.error_details_pb2 import RetryInfo @@ -31,11 +34,17 @@ from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue from opentelemetry.proto.resource.v1.resource_pb2 import Resource +from opentelemetry.sdk.resources import Resource as SDKResource logger = logging.getLogger(__name__) +SDKDataT = TypeVar("SDKDataT") +ResourceDataT = TypeVar("ResourceDataT") +TypingResourceT = TypeVar("TypingResourceT") +ExportServiceRequestT = TypeVar("ExportServiceRequestT") +ExportResultT = TypeVar("ExportResultT") -def _translate_key_values(key, value): +def _translate_key_values(key: Text, value: Any) -> KeyValue: if isinstance(value, bool): any_value = AnyValue(bool_value=value) @@ -64,8 +73,12 @@ def _translate_key_values(key, value): def _get_resource_data( - sdk_resource_instrumentation_library_data, resource_class, name -): + sdk_resource_instrumentation_library_data: Dict[ + SDKResource, ResourceDataT + ], + resource_class: Callable[..., TypingResourceT], + name: str, +) -> List[TypingResourceT]: resource_data = [] @@ -101,7 +114,9 @@ def _get_resource_data( # pylint: disable=no-member -class OTLPExporterMixin(ABC): +class OTLPExporterMixin( + ABC, Generic[SDKDataT, ExportServiceRequestT, ExportResultT] +): """OTLP span/metric exporter Args: @@ -114,7 +129,7 @@ def __init__( self, endpoint: str = "localhost:55680", credentials: ChannelCredentials = None, - metadata: tuple = None, + metadata: Optional[Tuple[Any]] = None, ): super().__init__() @@ -127,10 +142,12 @@ def __init__( self._client = self._stub(secure_channel(endpoint, credentials)) @abstractmethod - def _translate_data(self, data): + def _translate_data( + self, data: TypingSequence[SDKDataT] + ) -> ExportServiceRequestT: pass - def _export(self, data): + def _export(self, data: TypingSequence[SDKDataT]) -> ExportResultT: # expo returns a generator that yields delay values which grow # exponentially. Once delay is greater than max_value, the yielded # value will remain constant. @@ -190,5 +207,5 @@ def _export(self, data): return self._result.FAILURE - def shutdown(self): + def shutdown(self) -> None: pass diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py index 944428e37d..033de0d6dd 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py @@ -15,13 +15,14 @@ """OTLP Metrics Exporter""" import logging -from typing import Sequence +from typing import List, Sequence, Type, TypeVar, Union # pylint: disable=duplicate-code from opentelemetry.exporter.otlp.exporter import ( OTLPExporterMixin, _get_resource_data, ) +from opentelemetry.metrics import InstrumentT from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( ExportMetricsServiceRequest, ) @@ -41,9 +42,8 @@ MetricDescriptor, ResourceMetrics, ) -from opentelemetry.sdk.metrics import Counter -from opentelemetry.sdk.metrics import Metric as SDKMetric from opentelemetry.sdk.metrics import ( + Counter, SumObserver, UpDownCounter, UpDownSumObserver, @@ -51,14 +51,18 @@ ValueRecorder, ) from opentelemetry.sdk.metrics.export import ( + MetricRecord, MetricsExporter, MetricsExportResult, ) logger = logging.getLogger(__name__) +DataPointT = TypeVar("DataPointT", Int64DataPoint, DoubleDataPoint) -def _get_data_points(sdk_metric, data_point_class): +def _get_data_points( + sdk_metric: MetricRecord, data_point_class: Type[DataPointT] +) -> List[DataPointT]: data_points = [] @@ -89,7 +93,9 @@ def _get_data_points(sdk_metric, data_point_class): return data_points -def _get_temporality(instrument): +def _get_temporality( + instrument: InstrumentT, +) -> "MetricDescriptor.TemporalityValue": # pylint: disable=no-member if isinstance(instrument, (Counter, UpDownCounter)): temporality = MetricDescriptor.Temporality.DELTA @@ -107,12 +113,12 @@ def _get_temporality(instrument): return temporality -def _get_type(value_type): +def _get_type(value_type: Union[int, float]) -> "MetricDescriptor.TypeValue": # pylint: disable=no-member - if value_type is int: + if value_type is int: # type: ignore[comparison-overlap] type_ = MetricDescriptor.Type.INT64 - elif value_type is float: + elif value_type is float: # type: ignore[comparison-overlap] type_ = MetricDescriptor.Type.DOUBLE # FIXME What are the types that correspond with @@ -126,7 +132,13 @@ def _get_type(value_type): return type_ -class OTLPMetricsExporter(MetricsExporter, OTLPExporterMixin): +class OTLPMetricsExporter( + MetricsExporter, + OTLPExporterMixin[ + MetricRecord, ExportMetricsServiceRequest, MetricsExportResult + ], +): + # pylint: disable=unsubscriptable-object """OTLP metrics exporter Args: @@ -138,7 +150,9 @@ class OTLPMetricsExporter(MetricsExporter, OTLPExporterMixin): _stub = MetricsServiceStub _result = MetricsExportResult - def _translate_data(self, data): + def _translate_data( + self, data: Sequence[MetricRecord] + ) -> ExportMetricsServiceRequest: # pylint: disable=too-many-locals,no-member # pylint: disable=attribute-defined-outside-init @@ -193,6 +207,6 @@ def _translate_data(self, data): ) ) - def export(self, metrics: Sequence[SDKMetric]) -> MetricsExportResult: + def export(self, metrics: Sequence[MetricRecord]) -> MetricsExportResult: # pylint: disable=arguments-differ return self._export(metrics) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py index fd1d8e235e..c08d6049e1 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -41,7 +41,11 @@ # pylint: disable=no-member -class OTLPSpanExporter(SpanExporter, OTLPExporterMixin): +class OTLPSpanExporter( + SpanExporter, + OTLPExporterMixin[SDKSpan, ExportTraceServiceRequest, SpanExportResult], +): + # pylint: disable=unsubscriptable-object """OTLP span exporter Args: @@ -53,34 +57,34 @@ class OTLPSpanExporter(SpanExporter, OTLPExporterMixin): _result = SpanExportResult _stub = TraceServiceStub - def _translate_name(self, sdk_span): + def _translate_name(self, sdk_span: SDKSpan) -> None: self._collector_span_kwargs["name"] = sdk_span.name - def _translate_start_time(self, sdk_span): + def _translate_start_time(self, sdk_span: SDKSpan) -> None: self._collector_span_kwargs[ "start_time_unix_nano" ] = sdk_span.start_time - def _translate_end_time(self, sdk_span): + def _translate_end_time(self, sdk_span: SDKSpan) -> None: self._collector_span_kwargs["end_time_unix_nano"] = sdk_span.end_time - def _translate_span_id(self, sdk_span): + def _translate_span_id(self, sdk_span: SDKSpan) -> None: self._collector_span_kwargs[ "span_id" ] = sdk_span.context.span_id.to_bytes(8, "big") - def _translate_trace_id(self, sdk_span): + def _translate_trace_id(self, sdk_span: SDKSpan) -> None: self._collector_span_kwargs[ "trace_id" ] = sdk_span.context.trace_id.to_bytes(16, "big") - def _translate_parent(self, sdk_span): + def _translate_parent(self, sdk_span: SDKSpan) -> None: if sdk_span.parent is not None: self._collector_span_kwargs[ "parent_span_id" ] = sdk_span.parent.span_id.to_bytes(8, "big") - def _translate_context_trace_state(self, sdk_span): + def _translate_context_trace_state(self, sdk_span: SDKSpan) -> None: if sdk_span.context.trace_state is not None: self._collector_span_kwargs["trace_state"] = ",".join( [ @@ -89,7 +93,7 @@ def _translate_context_trace_state(self, sdk_span): ] ) - def _translate_attributes(self, sdk_span): + def _translate_attributes(self, sdk_span: SDKSpan) -> None: if sdk_span.attributes: self._collector_span_kwargs["attributes"] = [] @@ -103,7 +107,7 @@ def _translate_attributes(self, sdk_span): except Exception as error: # pylint: disable=broad-except logger.exception(error) - def _translate_events(self, sdk_span): + def _translate_events(self, sdk_span: SDKSpan) -> None: if sdk_span.events: self._collector_span_kwargs["events"] = [] @@ -127,7 +131,7 @@ def _translate_events(self, sdk_span): collector_span_event ) - def _translate_links(self, sdk_span): + def _translate_links(self, sdk_span: SDKSpan) -> None: if sdk_span.links: self._collector_span_kwargs["links"] = [] @@ -153,14 +157,17 @@ def _translate_links(self, sdk_span): collector_span_link ) - def _translate_status(self, sdk_span): + def _translate_status(self, sdk_span: SDKSpan) -> None: if sdk_span.status is not None: self._collector_span_kwargs["status"] = Status( code=sdk_span.status.canonical_code.value, message=sdk_span.status.description, ) - def _translate_data(self, data) -> ExportTraceServiceRequest: + def _translate_data( + self, data: Sequence[SDKSpan] + ) -> ExportTraceServiceRequest: + # pylint: disable=attribute-defined-outside-init sdk_resource_instrumentation_library_spans = {} From 12e42ac9c2847b35c2befa188786a08172aba6e8 Mon Sep 17 00:00:00 2001 From: alrex Date: Wed, 30 Sep 2020 07:50:42 -0700 Subject: [PATCH 11/15] Add support for OTLP v0.5.0 (#1143) --- .../opentelemetry-exporter-otlp/CHANGELOG.md | 3 + .../otlp/metrics_exporter/__init__.py | 170 ++++++++++-------- .../exporter/otlp/trace_exporter/__init__.py | 3 +- .../tests/test_otlp_metric_exporter.py | 53 +++--- .../tests/test_otlp_trace_exporter.py | 19 +- 5 files changed, 140 insertions(+), 108 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index a3686e1a10..8070b1b118 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Update OpenTelemetry protos to v0.5.0 + ([#1143](https://github.com/open-telemetry/opentelemetry-python/pull/1143)) + ## Version 0.13b0 Released 2020-09-17 diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py index 033de0d6dd..08a47c601e 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py @@ -15,14 +15,13 @@ """OTLP Metrics Exporter""" import logging -from typing import List, Sequence, Type, TypeVar, Union +from typing import List, Sequence, Type, TypeVar # pylint: disable=duplicate-code from opentelemetry.exporter.otlp.exporter import ( OTLPExporterMixin, _get_resource_data, ) -from opentelemetry.metrics import InstrumentT from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( ExportMetricsServiceRequest, ) @@ -31,17 +30,17 @@ ) from opentelemetry.proto.common.v1.common_pb2 import StringKeyValue from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + AggregationTemporality, DoubleDataPoint, + DoubleGauge, + DoubleSum, InstrumentationLibraryMetrics, - Int64DataPoint, -) -from opentelemetry.proto.metrics.v1.metrics_pb2 import ( - Metric as CollectorMetric, -) -from opentelemetry.proto.metrics.v1.metrics_pb2 import ( - MetricDescriptor, - ResourceMetrics, + IntDataPoint, + IntGauge, + IntSum, ) +from opentelemetry.proto.metrics.v1.metrics_pb2 import Metric as OTLPMetric +from opentelemetry.proto.metrics.v1.metrics_pb2 import ResourceMetrics from opentelemetry.sdk.metrics import ( Counter, SumObserver, @@ -57,7 +56,7 @@ ) logger = logging.getLogger(__name__) -DataPointT = TypeVar("DataPointT", Int64DataPoint, DoubleDataPoint) +DataPointT = TypeVar("DataPointT", IntDataPoint, DoubleDataPoint) def _get_data_points( @@ -93,45 +92,6 @@ def _get_data_points( return data_points -def _get_temporality( - instrument: InstrumentT, -) -> "MetricDescriptor.TemporalityValue": - # pylint: disable=no-member - if isinstance(instrument, (Counter, UpDownCounter)): - temporality = MetricDescriptor.Temporality.DELTA - elif isinstance(instrument, (ValueRecorder, ValueObserver)): - temporality = MetricDescriptor.Temporality.INSTANTANEOUS - elif isinstance(instrument, (SumObserver, UpDownSumObserver)): - temporality = MetricDescriptor.Temporality.CUMULATIVE - else: - raise Exception( - "No temporality defined for instrument type {}".format( - type(instrument) - ) - ) - - return temporality - - -def _get_type(value_type: Union[int, float]) -> "MetricDescriptor.TypeValue": - # pylint: disable=no-member - if value_type is int: # type: ignore[comparison-overlap] - type_ = MetricDescriptor.Type.INT64 - - elif value_type is float: # type: ignore[comparison-overlap] - type_ = MetricDescriptor.Type.DOUBLE - - # FIXME What are the types that correspond with - # MetricDescriptor.Type.HISTOGRAM and - # MetricDescriptor.Type.SUMMARY? - else: - raise Exception( - "No type defined for valie type {}".format(type(value_type)) - ) - - return type_ - - class OTLPMetricsExporter( MetricsExporter, OTLPExporterMixin[ @@ -150,6 +110,7 @@ class OTLPMetricsExporter( _stub = MetricsServiceStub _result = MetricsExportResult + # pylint: disable=no-self-use def _translate_data( self, data: Sequence[MetricRecord] ) -> ExportMetricsServiceRequest: @@ -158,6 +119,22 @@ def _translate_data( sdk_resource_instrumentation_library_metrics = {} + # The criteria to decide how to translate data is based on this table + # taken directly from OpenTelemetry Proto v0.5.0: + + # TODO: Update table after the decision on: + # https://github.com/open-telemetry/opentelemetry-specification/issues/731. + # By default, metrics recording using the OpenTelemetry API are exported as + # (the table does not include MeasurementValueType to avoid extra rows): + # + # Instrument Type + # ---------------------------------------------- + # Counter Sum(aggregation_temporality=delta;is_monotonic=true) + # UpDownCounter Sum(aggregation_temporality=delta;is_monotonic=false) + # ValueRecorder TBD + # SumObserver Sum(aggregation_temporality=cumulative;is_monotonic=true) + # UpDownSumObserver Sum(aggregation_temporality=cumulative;is_monotonic=false) + # ValueObserver Gauge() for sdk_metric in data: if sdk_metric.instrument.meter.resource not in ( @@ -167,37 +144,90 @@ def _translate_data( sdk_metric.instrument.meter.resource ] = InstrumentationLibraryMetrics() - self._metric_descriptor_kwargs = {} + type_class = { + int: { + "sum": {"class": IntSum, "argument": "int_sum"}, + "gauge": {"class": IntGauge, "argument": "int_gauge"}, + "data_point_class": IntDataPoint, + }, + float: { + "sum": {"class": DoubleSum, "argument": "double_sum"}, + "gauge": { + "class": DoubleGauge, + "argument": "double_gauge", + }, + "data_point_class": DoubleDataPoint, + }, + } + + value_type = sdk_metric.instrument.value_type + + sum_class = type_class[value_type]["sum"]["class"] + gauge_class = type_class[value_type]["gauge"]["class"] + data_point_class = type_class[value_type]["data_point_class"] + + if isinstance(sdk_metric.instrument, Counter): + otlp_metric_data = sum_class( + data_points=_get_data_points(sdk_metric, data_point_class), + aggregation_temporality=( + AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + ), + is_monotonic=True, + ) + argument = type_class[value_type]["sum"]["argument"] - metric_descriptor = MetricDescriptor( - name=sdk_metric.instrument.name, - description=sdk_metric.instrument.description, - unit=sdk_metric.instrument.unit, - type=_get_type(sdk_metric.instrument.value_type), - temporality=_get_temporality(sdk_metric.instrument), - ) + elif isinstance(sdk_metric.instrument, UpDownCounter): + otlp_metric_data = sum_class( + data_points=_get_data_points(sdk_metric, data_point_class), + aggregation_temporality=( + AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + ), + is_monotonic=False, + ) + argument = type_class[value_type]["sum"]["argument"] - if metric_descriptor.type == MetricDescriptor.Type.INT64: + elif isinstance(sdk_metric.instrument, (ValueRecorder)): + logger.warning("Skipping exporting of ValueRecorder metric") + continue - collector_metric = CollectorMetric( - metric_descriptor=metric_descriptor, - int64_data_points=_get_data_points( - sdk_metric, Int64DataPoint + elif isinstance(sdk_metric.instrument, SumObserver): + otlp_metric_data = sum_class( + data_points=_get_data_points(sdk_metric, data_point_class), + aggregation_temporality=( + AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE ), + is_monotonic=True, ) + argument = type_class[value_type]["sum"]["argument"] - elif metric_descriptor.type == MetricDescriptor.Type.DOUBLE: - - collector_metric = CollectorMetric( - metric_descriptor=metric_descriptor, - double_data_points=_get_data_points( - sdk_metric, DoubleDataPoint + elif isinstance(sdk_metric.instrument, UpDownSumObserver): + otlp_metric_data = sum_class( + data_points=_get_data_points(sdk_metric, data_point_class), + aggregation_temporality=( + AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE ), + is_monotonic=False, + ) + argument = type_class[value_type]["sum"]["argument"] + + elif isinstance(sdk_metric.instrument, (ValueObserver)): + otlp_metric_data = gauge_class( + data_points=_get_data_points(sdk_metric, data_point_class) ) + argument = type_class[value_type]["gauge"]["argument"] sdk_resource_instrumentation_library_metrics[ sdk_metric.instrument.meter.resource - ].metrics.append(collector_metric) + ].metrics.append( + OTLPMetric( + **{ + "name": sdk_metric.instrument.name, + "description": sdk_metric.instrument.description, + "unit": sdk_metric.instrument.unit, + argument: otlp_metric_data, + } + ) + ) return ExportMetricsServiceRequest( resource_metrics=_get_resource_data( diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py index c08d6049e1..e518716d39 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/trace_exporter/__init__.py @@ -208,7 +208,8 @@ def _translate_data( self._translate_status(sdk_span) self._collector_span_kwargs["kind"] = getattr( - CollectorSpan.SpanKind, sdk_span.kind.name + CollectorSpan.SpanKind, + "SPAN_KIND_{}".format(sdk_span.kind.name), ) sdk_resource_instrumentation_library_spans[ diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py index 20fecd44a2..1218fbbb33 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py @@ -25,18 +25,15 @@ StringKeyValue, ) from opentelemetry.proto.metrics.v1.metrics_pb2 import ( + AggregationTemporality, InstrumentationLibraryMetrics, - Int64DataPoint, -) -from opentelemetry.proto.metrics.v1.metrics_pb2 import ( - Metric as CollectorMetric, -) -from opentelemetry.proto.metrics.v1.metrics_pb2 import ( - MetricDescriptor, - ResourceMetrics, + IntDataPoint, + IntSum, ) +from opentelemetry.proto.metrics.v1.metrics_pb2 import Metric as OTLPMetric +from opentelemetry.proto.metrics.v1.metrics_pb2 import ResourceMetrics from opentelemetry.proto.resource.v1.resource_pb2 import ( - Resource as CollectorResource, + Resource as OTLPResource, ) from opentelemetry.sdk.metrics import Counter, MeterProvider from opentelemetry.sdk.metrics.export import MetricRecord @@ -71,7 +68,7 @@ def test_translate_metrics(self): expected = ExportMetricsServiceRequest( resource_metrics=[ ResourceMetrics( - resource=CollectorResource( + resource=OTLPResource( attributes=[ KeyValue(key="a", value=AnyValue(int_value=1)), KeyValue( @@ -82,26 +79,26 @@ def test_translate_metrics(self): instrumentation_library_metrics=[ InstrumentationLibraryMetrics( metrics=[ - CollectorMetric( - metric_descriptor=MetricDescriptor( - name="a", - description="b", - unit="c", - type=MetricDescriptor.Type.INT64, - temporality=( - MetricDescriptor.Temporality.DELTA + OTLPMetric( + name="a", + description="b", + unit="c", + int_sum=IntSum( + data_points=[ + IntDataPoint( + labels=[ + StringKeyValue( + key="a", value="b" + ) + ], + value=1, + ) + ], + aggregation_temporality=( + AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA ), + is_monotonic=True, ), - int64_data_points=[ - Int64DataPoint( - labels=[ - StringKeyValue( - key="a", value="b" - ) - ], - value=1, - ) - ], ) ] ) diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index 06a3877b92..dcc0239798 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -36,13 +36,13 @@ KeyValue, ) from opentelemetry.proto.resource.v1.resource_pb2 import ( - Resource as CollectorResource, + Resource as OTLPResource, ) from opentelemetry.proto.trace.v1.trace_pb2 import ( InstrumentationLibrarySpans, ResourceSpans, ) -from opentelemetry.proto.trace.v1.trace_pb2 import Span as CollectorSpan +from opentelemetry.proto.trace.v1.trace_pb2 import Span as OTLPSpan from opentelemetry.proto.trace.v1.trace_pb2 import Status from opentelemetry.sdk.resources import Resource as SDKResource from opentelemetry.sdk.trace import Span, TracerProvider @@ -51,7 +51,6 @@ SpanExportResult, ) from opentelemetry.sdk.util.instrumentation import InstrumentationInfo -from opentelemetry.trace import SpanKind class TraceServiceServicerUNAVAILABLEDelay(TraceServiceServicer): @@ -143,7 +142,7 @@ def setUp(self): "context.trace_id": 1, "context.span_id": 2, "attributes": OrderedDict([("a", 1), ("b", False)]), - "kind": SpanKind.INTERNAL, + "kind": OTLPSpan.SpanKind.SPAN_KIND_INTERNAL, # pylint: disable=no-member } ) ], @@ -207,7 +206,7 @@ def test_translate_spans(self): expected = ExportTraceServiceRequest( resource_spans=[ ResourceSpans( - resource=CollectorResource( + resource=OTLPResource( attributes=[ KeyValue(key="a", value=AnyValue(int_value=1)), KeyValue( @@ -221,7 +220,7 @@ def test_translate_spans(self): name="name", version="version" ), spans=[ - CollectorSpan( + OTLPSpan( # pylint: disable=no-member name="a", start_time_unix_nano=self.span.start_time, @@ -238,7 +237,9 @@ def test_translate_spans(self): parent_span_id=( b"\000\000\000\000\000\00009" ), - kind=CollectorSpan.SpanKind.INTERNAL, + kind=( + OTLPSpan.SpanKind.SPAN_KIND_INTERNAL + ), attributes=[ KeyValue( key="a", @@ -250,7 +251,7 @@ def test_translate_spans(self): ), ], events=[ - CollectorSpan.Event( + OTLPSpan.Event( name="a", time_unix_nano=1591240820506462784, attributes=[ @@ -271,7 +272,7 @@ def test_translate_spans(self): ], status=Status(code=0, message=""), links=[ - CollectorSpan.Link( + OTLPSpan.Link( trace_id=int.to_bytes( 1, 16, "big" ), From 9797954949ce7e1d302ddbece8793ab3182b5ff2 Mon Sep 17 00:00:00 2001 From: Amos Law Date: Tue, 6 Oct 2020 17:44:41 -0400 Subject: [PATCH 12/15] Protect access to Span implementation (#1188) --- .../tests/test_otlp_trace_exporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py index dcc0239798..e8c449c9df 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_trace_exporter.py @@ -45,7 +45,7 @@ from opentelemetry.proto.trace.v1.trace_pb2 import Span as OTLPSpan from opentelemetry.proto.trace.v1.trace_pb2 import Status from opentelemetry.sdk.resources import Resource as SDKResource -from opentelemetry.sdk.trace import Span, TracerProvider +from opentelemetry.sdk.trace import TracerProvider, _Span from opentelemetry.sdk.trace.export import ( SimpleExportSpanProcessor, SpanExportResult, @@ -123,7 +123,7 @@ def setUp(self): type(event_mock).name = PropertyMock(return_value="a") - self.span = Span( + self.span = _Span( "a", context=Mock( **{ From bd6d453bf15223d799430ddb4b937ace56393848 Mon Sep 17 00:00:00 2001 From: alrex Date: Wed, 7 Oct 2020 09:07:49 -0700 Subject: [PATCH 13/15] Adding Resource to MetricRecord (#1209) Co-authored-by: Diego Hurtado --- .../exporter/otlp/metrics_exporter/__init__.py | 6 +++--- .../tests/test_otlp_metric_exporter.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py index 08a47c601e..1fa1bf24f1 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py @@ -137,11 +137,11 @@ def _translate_data( # ValueObserver Gauge() for sdk_metric in data: - if sdk_metric.instrument.meter.resource not in ( + if sdk_metric.resource not in ( sdk_resource_instrumentation_library_metrics.keys() ): sdk_resource_instrumentation_library_metrics[ - sdk_metric.instrument.meter.resource + sdk_metric.resource ] = InstrumentationLibraryMetrics() type_class = { @@ -217,7 +217,7 @@ def _translate_data( argument = type_class[value_type]["gauge"]["argument"] sdk_resource_instrumentation_library_metrics[ - sdk_metric.instrument.meter.resource + sdk_metric.resource ].metrics.append( OTLPMetric( **{ diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py index 1218fbbb33..1eba2bef66 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py @@ -44,20 +44,19 @@ class TestOTLPMetricExporter(TestCase): def setUp(self): self.exporter = OTLPMetricsExporter() - + resource = SDKResource(OrderedDict([("a", 1), ("b", False)])) self.counter_metric_record = MetricRecord( Counter( "a", "b", "c", int, - MeterProvider( - resource=SDKResource(OrderedDict([("a", 1), ("b", False)])) - ).get_meter(__name__), + MeterProvider(resource=resource,).get_meter(__name__), ("d",), ), OrderedDict([("e", "f")]), SumAggregator(), + resource, ) def test_translate_metrics(self): From 94458989c2acd0907d0d01f188b9bb58f4ab019c Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Mon, 12 Oct 2020 09:09:07 -0600 Subject: [PATCH 14/15] Add timestamps to aggregators and OTLP metrics exporter (#1199) --- exporter/opentelemetry-exporter-otlp/CHANGELOG.md | 2 ++ .../exporter/otlp/metrics_exporter/__init__.py | 6 ++++++ .../tests/test_otlp_metric_exporter.py | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 8070b1b118..9c807d3225 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add timestamps to OTLP exporter + ([#1199](https://github.com/open-telemetry/opentelemetry-python/pull/1199)) - Update OpenTelemetry protos to v0.5.0 ([#1143](https://github.com/open-telemetry/opentelemetry-python/pull/1143)) diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py index 1fa1bf24f1..40feb222fb 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/metrics_exporter/__init__.py @@ -85,6 +85,12 @@ def _get_data_points( data_point_class( labels=string_key_values, value=view_data.aggregator.current, + start_time_unix_nano=( + view_data.aggregator.last_checkpoint_timestamp + ), + time_unix_nano=( + view_data.aggregator.last_update_timestamp + ), ) ) break diff --git a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py index 1eba2bef66..21a718b84e 100644 --- a/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py +++ b/exporter/opentelemetry-exporter-otlp/tests/test_otlp_metric_exporter.py @@ -14,6 +14,7 @@ from collections import OrderedDict from unittest import TestCase +from unittest.mock import patch from opentelemetry.exporter.otlp.metrics_exporter import OTLPMetricsExporter from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( @@ -59,9 +60,12 @@ def setUp(self): resource, ) - def test_translate_metrics(self): + @patch("opentelemetry.sdk.metrics.export.aggregate.time_ns") + def test_translate_metrics(self, mock_time_ns): # pylint: disable=no-member + mock_time_ns.configure_mock(**{"return_value": 1}) + self.counter_metric_record.instrument.add(1, OrderedDict([("a", "b")])) expected = ExportMetricsServiceRequest( @@ -91,6 +95,7 @@ def test_translate_metrics(self): ) ], value=1, + time_unix_nano=1, ) ], aggregation_temporality=( From 48e166c85988b5d48ce24ace3893ab982647c599 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 13 Oct 2020 14:38:09 -0400 Subject: [PATCH 15/15] chore: bump dev version (#1235) --- exporter/opentelemetry-exporter-otlp/CHANGELOG.md | 4 ++++ exporter/opentelemetry-exporter-otlp/setup.cfg | 6 +++--- .../src/opentelemetry/exporter/otlp/version.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md index 9c807d3225..479f2105e7 100644 --- a/exporter/opentelemetry-exporter-otlp/CHANGELOG.md +++ b/exporter/opentelemetry-exporter-otlp/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## Version 0.14b0 + +Released 2020-10-13 + - Add timestamps to OTLP exporter ([#1199](https://github.com/open-telemetry/opentelemetry-python/pull/1199)) - Update OpenTelemetry protos to v0.5.0 diff --git a/exporter/opentelemetry-exporter-otlp/setup.cfg b/exporter/opentelemetry-exporter-otlp/setup.cfg index 12fdd62775..0f5870813e 100644 --- a/exporter/opentelemetry-exporter-otlp/setup.cfg +++ b/exporter/opentelemetry-exporter-otlp/setup.cfg @@ -41,9 +41,9 @@ packages=find_namespace: install_requires = grpcio >= 1.0.0, < 2.0.0 googleapis-common-protos ~= 1.52.0 - opentelemetry-api == 0.14.dev0 - opentelemetry-sdk == 0.14.dev0 - opentelemetry-proto == 0.14.dev0 + opentelemetry-api == 0.15.dev0 + opentelemetry-sdk == 0.15.dev0 + opentelemetry-proto == 0.15.dev0 backoff ~= 1.10.0 [options.extras_require] diff --git a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py index 0f99027898..e7b342d644 100644 --- a/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py +++ b/exporter/opentelemetry-exporter-otlp/src/opentelemetry/exporter/otlp/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.14.dev0" +__version__ = "0.15.dev0"