From 988890e5c5c9cc9467570ee923494d58dac0b8d8 Mon Sep 17 00:00:00 2001 From: Nathan Button Date: Wed, 18 Sep 2019 09:01:17 -0700 Subject: [PATCH 1/5] A trace exporter for datadog based off of the go exporter https://github.com/DataDog/opencensus-go-exporter-datadog WIP - believe that the barebones exportor is working Working traces Clean up move files back to where they go clean up todo fix folder name WIP - start work on getting PR ready test, lint, and coverage passing clean up clean up --- .../examples/datadog.py | 22 ++ .../opencensus/__init__.py | 1 + .../opencensus/ext/__init__.py | 1 + .../opencensus/ext/datadog/__init__.py | 1 + .../opencensus/ext/datadog/traces.py | 372 ++++++++++++++++++ .../opencensus/ext/datadog/transport.py | 45 +++ contrib/opencensus-ext-datadog/setup.py | 50 +++ .../tests/traces_test.py | 350 ++++++++++++++++ .../tests/transport_test.py | 14 + noxfile.py | 1 + 10 files changed, 857 insertions(+) create mode 100644 contrib/opencensus-ext-datadog/examples/datadog.py create mode 100644 contrib/opencensus-ext-datadog/opencensus/__init__.py create mode 100644 contrib/opencensus-ext-datadog/opencensus/ext/__init__.py create mode 100644 contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py create mode 100644 contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py create mode 100644 contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py create mode 100644 contrib/opencensus-ext-datadog/setup.py create mode 100644 contrib/opencensus-ext-datadog/tests/traces_test.py create mode 100644 contrib/opencensus-ext-datadog/tests/transport_test.py diff --git a/contrib/opencensus-ext-datadog/examples/datadog.py b/contrib/opencensus-ext-datadog/examples/datadog.py new file mode 100644 index 000000000..de1fcde36 --- /dev/null +++ b/contrib/opencensus-ext-datadog/examples/datadog.py @@ -0,0 +1,22 @@ +from flask import Flask +from opencensus.ext.flask.flask_middleware import FlaskMiddleware +from opencensus.trace.samplers import AlwaysOnSampler +from traces import DatadogTaceExporter +from traces import Options + +app = Flask(__name__) +middleware = FlaskMiddleware(app, + blacklist_paths=['/healthz'], + sampler=AlwaysOnSampler(), + exporter=DatadogTaceExporter( + Options(service='python-export-test', + global_tags={"stack": "example"}))) + + +@app.route('/') +def hello(): + return 'Hello World!' + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080, threaded=True) diff --git a/contrib/opencensus-ext-datadog/opencensus/__init__.py b/contrib/opencensus-ext-datadog/opencensus/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py b/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py new file mode 100644 index 000000000..5c03b1a2a --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py @@ -0,0 +1,372 @@ +# Copyright 2018, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +import codecs +from datetime import datetime +import bitarray +from opencensus.common.transports import sync +from opencensus.trace import base_exporter +from opencensus.trace import span_data +from opencensus.ext.datadog.transport import DDTransport + + +class Options(object): + """ Options contains options for configuring the exporter. + The address can be empty as the prometheus client will + assume it's localhost + + :type namespace: str + :param namespace: Namespace specifies the namespaces to which metric keys + are appended. Defaults to ''. + + :type service: str + :param service: service specifies the service name used for tracing. + + :type trace_addr: str + :param trace_addr: trace_addr specifies the host[:port] address of the + Datadog Trace Agent. It defaults to localhost:8126 + + :type global_tags: dict + :param global_tags: global_tags is a set of tags that will be + applied to all exported spans. + """ + def __init__(self, service='', trace_addr='localhost:8126', + global_tags={}): + self._service = service + self._trace_addr = trace_addr + for k, v in global_tags.items(): + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError( + "global tags must be dictionary of string string") + self._global_tags = global_tags + + @property + def trace_addr(self): + """ specifies the host[:port] address of the Datadog Trace Agent. + """ + return self._trace_addr + + @property + def service(self): + """ Specifies the service name used for tracing. + """ + return self._service + + @property + def global_tags(self): + """ Specifies the namespaces to which metric keys are appended + """ + return self._global_tags + + +class DatadogTaceExporter(base_exporter.Exporter): + """ A exporter that send traces and trace spans to Datadog. + + :type options: :class:`~opencensus.ext.datadog.Options` + :param options: An options object with the parameters to instantiate the + Datadog Exporter. + + :type transport: + :class:`opencensus.common.transports.sync.SyncTransport` or + :class:`opencensus.common.transports.async_.AsyncTransport` + :param transport: An instance of a Transport to send data with. + """ + def __init__(self, options, transport=sync.SyncTransport): + self._options = options + self._transport = transport(self) + self._dd_transport = DDTransport(options.trace_addr) + + @property + def transport(self): + """ The transport way to be sent data to server + (default is sync). + """ + return self._transport + + @property + def options(self): + """ Options to be used to configure the exporter + """ + return self._options + + def export(self, span_datas): + """ + :type span_datas: list of :class: + `~opencensus.trace.span_data.SpanData` + :param list of opencensus.trace.span_data.SpanData span_datas: + SpanData tuples to export + """ + if span_datas is not None: # pragma: NO COVER + self.transport.export(span_datas) + + def emit(self, span_datas): + """ + :type span_datas: list of :class: + `~opencensus.trace.span_data.SpanData` + :param list of opencensus.trace.span_data.SpanData span_datas: + SpanData tuples to emit + """ + # Map each span data to it's corresponding trace id + trace_span_map = defaultdict(list) + for sd in span_datas: + trace_span_map[sd.context.trace_id] += [sd] + + dd_spans = [] + # Write spans to Stackdriver + for _, sds in trace_span_map.items(): + # convert to the legacy trace json for easier refactoring + trace = span_data.format_legacy_trace_json(sds) + dd_spans.append(self.translate_to_datadog(trace)) + + self._dd_transport.send_traces(dd_spans) + + def translate_to_datadog(self, trace): + """Translate the spans json to Datadog format. + + :type trace: dict + :param trace: Trace dictionary + + :rtype: dict + :returns: Spans in Datadog Trace format. + """ + + spans_json = trace.get('spans') + trace_id = convert_id(trace.get('traceId')[8:]) + dd_trace = [] + for span in spans_json: + span_id_int = convert_id(span.get('spanId')) + # Set meta at the end. + meta = self.options.global_tags.copy() + + dd_span = { + 'span_id': span_id_int, + 'trace_id': trace_id, + 'name': "opencensus", + 'service': self.options.service, + 'resource': span.get("displayName").get("value"), + } + + start_time = datetime.strptime(span.get('startTime'), TIME_FMT) + + # The start time of the request in nanoseconds from the unix epoch. + epoch = datetime.utcfromtimestamp(0) + dd_span["start"] = int((start_time - epoch).total_seconds() * + 1000.0 * 1000.0 * 1000.0) + + end_time = datetime.strptime(span.get('endTime'), TIME_FMT) + duration_td = end_time - start_time + + # The duration of the request in nanoseconds. + dd_span["duration"] = int(duration_td.total_seconds() * 1000.0 * + 1000.0 * 1000.0) + + if span.get('parentSpanId') is not None: + parent_span_id = convert_id(span.get('parentSpanId')) + dd_span['parent_id'] = parent_span_id + + code = STATUS_CODES.get(span["status"].get("code")) + if code is None: + code = {} + code["message"] = "ERR_CODE_" + str(span["status"].get("code")) + code["status"] = 500 + + # opencensus.trace.span.SpanKind + dd_span['type'] = to_dd_type(span.get("kind")) + dd_span["error"] = 0 + if int(code.get("status") / 100) == 4 or int( + code.get("status") / 100) == 5: + dd_span["error"] = 1 + meta["error.type"] = code.get("message") + + if span.get("status").get("message") is not None: + meta["error.msg"] = span.get("status").get("message") + + meta["opencensus.status_code"] = str(code.get("status")) + meta["opencensus.status"] = code.get("message") + + if span.get("status").get("message") is not None: + meta["opencensus.status_description"] = span.get("status").get( + "message") + + atts = span.get("attributes").get("attributeMap") + atts_to_metadata(atts, meta=meta) + + dd_span["meta"] = meta + dd_trace.append(dd_span) + + return dd_trace + + +def atts_to_metadata(atts, meta={}): + """Translate the attributes to Datadog meta format. + + :type atts: dict + :param atts: Attributes dictionary + + :rtype: dict + :returns: meta dictionary + """ + for key, elem in atts.items(): + value = value_from_atts_elem(elem) + if value != "": + meta[key] = value + + return meta + + +def value_from_atts_elem(elem): + """ value_from_atts_elem takes an attribute element and retuns a string value + + :type elem: dict + :param elem: Element from the attributes map + + :rtype: str + :return: A string rep of the element value + """ + if elem.get('string_value') is not None: + return elem.get('string_value').get('value') + elif elem.get('int_value') is not None: + return str(elem.get('int_value')) + elif elem.get('bool_value') is not None: + return str(elem.get('bool_value')) + elif elem.get('double_value') is not None: + return str(elem.get('double_value').get('value')) + return "" + + +def to_dd_type(oc_kind): + """ to_dd_type takes an OC kind int ID and returns a dd string of the span type + + :type oc_kind: int + :param oc_kind: OC kind id + + :rtype: string + :returns: A string of the Span type. + """ + if oc_kind == 2: + return "client" + elif oc_kind == 1: + return "server" + else: + return "unspecified" + + +def new_trace_exporter(option): + """ new_trace_exporter returns an exporter + that exports traces to Datadog. + """ + if option.service == "": + raise ValueError("Service can not be empty string.") + + exporter = DatadogTaceExporter(options=option) + return exporter + + +def convert_id(str_id): + """ convert_id takes a string and converts that to an int that is no + more than 64 bits wide. It does this by first converting the string + to a bit array then taking up to the 64th bit and creating and int. + This is equivlent to the go-exporter ID converter + ` binary.BigEndian.Uint64(s.SpanContext.SpanID[:])` + + :type str_id: str + :param str_id: string id + + :rtype: int + :returns: An int that is no more than 64 bits wide + """ + id_bitarray = bitarray.bitarray(endian='big') + id_bitarray.frombytes(str_id.encode()) + cut_off = len(id_bitarray) + if cut_off > 64: + cut_off = 64 + id_cutoff_bytearray = id_bitarray[:cut_off].tobytes() + id_int = int(codecs.encode(id_cutoff_bytearray, 'hex'), 16) + return id_int + + +# 2019-09-19T14:05:15.808570Z +TIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" + +# https://opencensus.io/tracing/span/status/ +STATUS_CODES = { + 0: { + "message": "OK", + "status": int(200) + }, + 1: { + "message": "CANCELLED", + "status": int(499) + }, + 2: { + "message": "UNKNOWN", + "status": int(500) + }, + 3: { + "message": "INVALID_ARGUMENT", + "status": int(400) + }, + 4: { + "message": "DEADLINE_EXCEEDED", + "status": int(504) + }, + 5: { + "message": "NOT_FOUND", + "status": int(404) + }, + 6: { + "message": "ALREADY_EXISTS", + "status": int(409) + }, + 7: { + "message": "PERMISSION_DENIED", + "status": int(403) + }, + 8: { + "message": "RESOURCE_EXHAUSTED", + "status": int(429) + }, + 9: { + "message": "FAILED_PRECONDITION", + "status": int(400) + }, + 10: { + "message": "ABORTED", + "status": int(409) + }, + 11: { + "message": "OUT_OF_RANGE", + "status": int(400) + }, + 12: { + "message": "UNIMPLEMENTED", + "status": int(502) + }, + 13: { + "message": "INTERNAL", + "status": int(500) + }, + 14: { + "message": "UNAVAILABLE", + "status": int(503) + }, + 15: { + "message": "DATA_LOSS", + "status": int(501) + }, + 16: { + "message": "UNAUTHENTICATED", + "status": int(401) + }, +} diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py new file mode 100644 index 000000000..72a598a77 --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py @@ -0,0 +1,45 @@ +import platform +import requests + + +class DDTransport(object): + """ DDTransport contains all the logic for sending Traces to Datadog + + :type trace_addr: str + :param trace_addr: trace_addr specifies the host[:port] address of the + Datadog Trace Agent. + """ + def __init__(self, trace_addr): + self._trace_addr = trace_addr + + self._headers = { + "Datadog-Meta-Lang": "python", + "Datadog-Meta-Lang-Interpreter": platform.platform(), + # Following the example of the Golang version it is prefixed + # OC for Opencensus. + "Datadog-Meta-Tracer-Version": "OC/0.0.1", + "Content-Type": "application/json", + } + + @property + def trace_addr(self): + """ specifies the host[:port] address of the Datadog Trace Agent. + """ + return self._trace_addr + + @property + def headers(self): + """ specifies the headers that will be attached to HTTP request sent to DD. + """ + return self._headers + + def send_traces(self, trace): + """ Sends traces to the Datadog Tracing Agent + + :type trace: dic + :param trace: Trace dictionary + """ + + requests.post("http://" + self.trace_addr + "/v0.4/traces", + json=trace, + headers=self.headers) diff --git a/contrib/opencensus-ext-datadog/setup.py b/contrib/opencensus-ext-datadog/setup.py new file mode 100644 index 000000000..8ac96d1c7 --- /dev/null +++ b/contrib/opencensus-ext-datadog/setup.py @@ -0,0 +1,50 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import find_packages +from setuptools import setup +from version import __version__ + +setup( + name='opencensus-ext-datadog', + version=__version__, # noqa + author='OpenCensus Authors', + author_email='census-developers@googlegroups.com', + classifiers=[ + 'Intended Audience :: Developers', + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + description='OpenCensus Datadog exporter', + include_package_data=True, + install_requires=[ + 'bitarray >= 1.0.1, < 2.0.0', + 'opencensus >= 0.8.dev0, < 1.0.0', + ], + extras_require={}, + license='Apache-2.0', + packages=find_packages(exclude=('examples', 'tests',)), + namespace_packages=[], + url='https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-datadog', # noqa: E501 + zip_safe=False, +) diff --git a/contrib/opencensus-ext-datadog/tests/traces_test.py b/contrib/opencensus-ext-datadog/tests/traces_test.py new file mode 100644 index 000000000..71d8644f1 --- /dev/null +++ b/contrib/opencensus-ext-datadog/tests/traces_test.py @@ -0,0 +1,350 @@ +import unittest +import mock +from opencensus.trace import span_data as span_data_module +from opencensus.trace import span_context +from opencensus.ext.datadog.traces import (convert_id, to_dd_type, + value_from_atts_elem, + atts_to_metadata, + new_trace_exporter, + DatadogTaceExporter, Options) + + +class TestTraces(unittest.TestCase): + def setUp(self): + pass + + def test_convert_id(self): + test_cases = [{ + 'input': 'd17b83f89a2cbb08c2fa4469', + 'expected': 0x6431376238336638, + }, { + 'input': '1ff346aeb5d12443', + 'expected': 0x3166663334366165, + }, { + 'input': '8c9b71d2ffb05ede97bea00a', + 'expected': 0x3863396237316432, + }, { + 'input': 'a3e1b9b4ce7d2e33', + 'expected': 0x6133653162396234, + }, { + 'input': '2f79a1a078c0a4d070094440', + 'expected': 0x3266373961316130, + }, { + 'input': '0018b3f50e44f875', + 'expected': 0x3030313862336635, + }, { + 'input': 'cba7b2832de221dbc1ac8e77', + 'expected': 0x6362613762323833, + }, { + 'input': 'a3e1b9b4', + 'expected': 0x6133653162396234, + }] + for tc in test_cases: + self.assertEqual(convert_id(tc['input']), tc['expected']) + + def test_to_dd_type(self): + self.assertEqual(to_dd_type(1), "server") + self.assertEqual(to_dd_type(2), "client") + self.assertEqual(to_dd_type(3), "unspecified") + + def test_value_from_atts_elem(self): + test_cases = [{ + 'elem': { + 'string_value': { + 'value': 'StringValue' + } + }, + 'expected': 'StringValue' + }, { + 'elem': { + 'int_value': 10 + }, + 'expected': '10' + }, { + 'elem': { + 'bool_value': True + }, + 'expected': 'True' + }, { + 'elem': { + 'bool_value': False + }, + 'expected': 'False' + }, { + 'elem': { + 'double_value': { + 'value': 2.1 + } + }, + 'expected': '2.1' + }, { + 'elem': { + 'somthing_les': 2.1 + }, + 'expected': '' + }] + + for tc in test_cases: + self.assertEqual(value_from_atts_elem(tc['elem']), tc['expected']) + + def test_export(self): + mock_dd_transport = mock.Mock() + exporter = DatadogTaceExporter(options=Options(), + transport=MockTransport) + exporter._dd_transport = mock_dd_transport + exporter.export({}) + self.assertTrue(exporter.transport.export_called) + + @mock.patch('opencensus.ext.datadog.traces.' + 'DatadogTaceExporter.translate_to_datadog', + return_value=None) + def test_emit(self, mr_mock): + + trace_id = '6e0c63257de34c92bf9efcd03927272e' + span_datas = [ + span_data_module.SpanData( + name='span', + context=span_context.SpanContext(trace_id=trace_id), + span_id=None, + parent_span_id=None, + attributes=None, + start_time=None, + end_time=None, + child_span_count=None, + stack_trace=None, + annotations=None, + message_events=None, + links=None, + status=None, + same_process_as_parent_span=None, + span_kind=0, + ) + ] + + mock_dd_transport = mock.Mock() + exporter = DatadogTaceExporter(options=Options(service="dd-unit-test"), + transport=MockTransport) + exporter._dd_transport = mock_dd_transport + + exporter.emit(span_datas) + # mock_dd_transport.send_traces.assert_called_with(datadog_spans) + self.assertTrue(mock_dd_transport.send_traces.called) + + def test_translate_to_datadog(self): + test_cases = [ + { + 'status': {'code': 0}, + 'prt_span_id': '6e0c63257de34c92', + 'expt_prt_span_id': 0x3665306336333235, + 'attributes': { + 'attributeMap': { + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'key_double': { + 'double_value': { + 'value': 123.45 + } + }, + 'http.host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' + } + } + } + }, + 'meta': { + 'key': 'value', + 'key_double': '123.45', + 'http.host': 'host', + 'opencensus.status': 'OK', + 'opencensus.status_code': '200' + }, + 'error': 0 + }, + { + 'status': {'code': 23}, + 'attributes': { + 'attributeMap': {} + }, + 'meta': { + 'error.type': 'ERR_CODE_23', + 'opencensus.status': 'ERR_CODE_23', + 'opencensus.status_code': '500' + }, + 'error': 1 + }, + { + 'status': {'code': 23, 'message': 'I_AM_A_TEAPOT'}, + 'attributes': { + 'attributeMap': {} + }, + 'meta': { + 'error.type': 'ERR_CODE_23', + 'opencensus.status': 'ERR_CODE_23', + 'opencensus.status_code': '500', + 'opencensus.status_description': 'I_AM_A_TEAPOT', + 'error.msg': 'I_AM_A_TEAPOT' + }, + 'error': 1 + }, + { + 'status': {'code': 0, 'message': 'OK'}, + 'attributes': { + 'attributeMap': {} + }, + 'meta': { + 'opencensus.status': 'OK', + 'opencensus.status_code': '200', + 'opencensus.status_description': 'OK' + }, + 'error': 0 + } + ] + trace_id = '6e0c63257de34c92bf9efcd03927272e' + expected_trace_id = 0x3764653334633932 + span_id = '6e0c63257de34c92' + expected_span_id = 0x3665306336333235 + span_name = 'test span' + start_time = '2019-09-19T14:05:15.000000Z' + start_time_epoch = 1568901915000000000 + end_time = '2019-09-19T14:05:16.000000Z' + span_duration = 1 * 1000 * 1000 * 1000 + + for tc in test_cases: + mock_dd_transport = mock.Mock() + opts = Options(service="dd-unit-test") + tran = MockTransport + exporter = DatadogTaceExporter(options=opts, transport=tran) + exporter._dd_transport = mock_dd_transport + trace = { + 'spans': [{ + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, + 'spanId': span_id, + 'startTime': start_time, + 'endTime': end_time, + 'parentSpanId': tc.get('prt_span_id'), + 'attributes': tc.get('attributes'), + 'someRandomKey': 'this should not be included in result', + 'childSpanCount': 0, + 'kind': 1, + 'status': tc.get('status') + }], + 'traceId': + trace_id, + } + + spans = list(exporter.translate_to_datadog(trace)) + expected_traces = [{ + 'span_id': expected_span_id, + 'trace_id': expected_trace_id, + 'name': 'opencensus', + 'service': 'dd-unit-test', + 'resource': span_name, + 'start': start_time_epoch, + 'duration': span_duration, + 'meta': tc.get('meta'), + 'type': 'server', + 'error': tc.get('error') + }] + + if tc.get('prt_span_id') is not None: + expected_traces[0]['parent_id'] = tc.get('expt_prt_span_id') + self.assertEqual.__self__.maxDiff = None + self.assertEqual(spans, expected_traces) + + def test_atts_to_metadata(self): + test_cases = [ + { + 'input': { + 'key_string': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'key_double': { + 'double_value': { + 'value': 123.45 + } + }, + }, + 'input_meta': {}, + 'output': { + 'key_string': 'value', + 'key_double': '123.45' + } + }, + { + 'input': { + 'key_string': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + }, + 'input_meta': { + 'key': 'in_meta' + }, + 'output': { + 'key_string': 'value', + 'key': 'in_meta' + } + }, + { + 'input': { + 'key_string': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'invalid': { + 'unknown_value': "na" + } + }, + 'input_meta': {}, + 'output': { + 'key_string': 'value', + } + } + ] + + for tc in test_cases: + out = atts_to_metadata(tc.get('input'), meta=tc.get('input_meta')) + self.assertEqual(out, tc.get('output')) + + def test_new_trace_exporter(self): + self.assertRaises(ValueError, new_trace_exporter, Options()) + try: + new_trace_exporter(Options(service="test")) + except ValueError: + self.fail("new_trace_exporter raised ValueError unexpectedly") + + def test_constructure(self): + self.assertRaises(TypeError, Options, global_tags={'int_bad': 1}) + try: + Options(global_tags={'good': 'tag'}) + except TypeError: + self.fail("Constructure raised TypeError unexpectedly") + + +class MockTransport(object): + def __init__(self, exporter=None): + self.export_called = False + self.exporter = exporter + + def export(self, trace): + self.export_called = True + + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/opencensus-ext-datadog/tests/transport_test.py b/contrib/opencensus-ext-datadog/tests/transport_test.py new file mode 100644 index 000000000..51838bcaa --- /dev/null +++ b/contrib/opencensus-ext-datadog/tests/transport_test.py @@ -0,0 +1,14 @@ +import unittest +import mock + +from opencensus.ext.datadog.transport import DDTransport + + +class TestTraces(unittest.TestCase): + def setUp(self): + pass + + @mock.patch('requests.post', return_value=None) + def test_send_traces(self, mr_mock): + transport = DDTransport('test') + transport.send_traces({}) diff --git a/noxfile.py b/noxfile.py index b65df590c..e419f2f89 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,6 +24,7 @@ def _install_dev_packages(session): session.install('-e', '.') session.install('-e', 'contrib/opencensus-ext-azure') + session.install('-e', 'contrib/opencensus-ext-datadog') session.install('-e', 'contrib/opencensus-ext-dbapi') session.install('-e', 'contrib/opencensus-ext-django') session.install('-e', 'contrib/opencensus-ext-flask') From 20e79baf6a413d98a6a893493b303fe822181b28 Mon Sep 17 00:00:00 2001 From: Nathan Button Date: Mon, 30 Sep 2019 11:11:31 -0700 Subject: [PATCH 2/5] update readme --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 87ddf495f..e056d4c4d 100644 --- a/README.rst +++ b/README.rst @@ -226,12 +226,14 @@ Trace Exporter -------------- - `Azure`_ +- `Datadog`_ - `Jaeger`_ - `OCAgent`_ - `Stackdriver`_ - `Zipkin`_ .. _Azure: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-azure +.. _Datadog: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-datadog .. _Django: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-django .. _Flask: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-flask .. _gevent: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-gevent From 1fb81c25160e4663a44e2ded3c07ed0efe4e0e51 Mon Sep 17 00:00:00 2001 From: Nathan Button Date: Mon, 30 Sep 2019 17:52:07 -0700 Subject: [PATCH 3/5] sort imports, fix typo, remove stray comment --- .../examples/datadog.py | 5 +++-- .../opencensus/ext/datadog/traces.py | 12 ++++++----- .../tests/traces_test.py | 21 +++++++++++-------- .../tests/transport_test.py | 1 + 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/contrib/opencensus-ext-datadog/examples/datadog.py b/contrib/opencensus-ext-datadog/examples/datadog.py index de1fcde36..7e1a6e0d0 100644 --- a/contrib/opencensus-ext-datadog/examples/datadog.py +++ b/contrib/opencensus-ext-datadog/examples/datadog.py @@ -1,14 +1,15 @@ from flask import Flask + from opencensus.ext.flask.flask_middleware import FlaskMiddleware from opencensus.trace.samplers import AlwaysOnSampler -from traces import DatadogTaceExporter +from traces import DatadogTraceExporter from traces import Options app = Flask(__name__) middleware = FlaskMiddleware(app, blacklist_paths=['/healthz'], sampler=AlwaysOnSampler(), - exporter=DatadogTaceExporter( + exporter=DatadogTraceExporter( Options(service='python-export-test', global_tags={"stack": "example"}))) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py index 5c03b1a2a..711c2c92a 100644 --- a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py @@ -11,14 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict import codecs +from collections import defaultdict from datetime import datetime + import bitarray + from opencensus.common.transports import sync +from opencensus.ext.datadog.transport import DDTransport from opencensus.trace import base_exporter from opencensus.trace import span_data -from opencensus.ext.datadog.transport import DDTransport class Options(object): @@ -70,7 +72,7 @@ def global_tags(self): return self._global_tags -class DatadogTaceExporter(base_exporter.Exporter): +class DatadogTraceExporter(base_exporter.Exporter): """ A exporter that send traces and trace spans to Datadog. :type options: :class:`~opencensus.ext.datadog.Options` @@ -123,7 +125,7 @@ def emit(self, span_datas): trace_span_map[sd.context.trace_id] += [sd] dd_spans = [] - # Write spans to Stackdriver + # Write spans to Datadog for _, sds in trace_span_map.items(): # convert to the legacy trace json for easier refactoring trace = span_data.format_legacy_trace_json(sds) @@ -269,7 +271,7 @@ def new_trace_exporter(option): if option.service == "": raise ValueError("Service can not be empty string.") - exporter = DatadogTaceExporter(options=option) + exporter = DatadogTraceExporter(options=option) return exporter diff --git a/contrib/opencensus-ext-datadog/tests/traces_test.py b/contrib/opencensus-ext-datadog/tests/traces_test.py index 71d8644f1..46f7e757b 100644 --- a/contrib/opencensus-ext-datadog/tests/traces_test.py +++ b/contrib/opencensus-ext-datadog/tests/traces_test.py @@ -1,12 +1,14 @@ import unittest + import mock -from opencensus.trace import span_data as span_data_module -from opencensus.trace import span_context + from opencensus.ext.datadog.traces import (convert_id, to_dd_type, value_from_atts_elem, atts_to_metadata, new_trace_exporter, - DatadogTaceExporter, Options) + DatadogTraceExporter, Options) +from opencensus.trace import span_data as span_data_module +from opencensus.trace import span_context class TestTraces(unittest.TestCase): @@ -89,14 +91,14 @@ def test_value_from_atts_elem(self): def test_export(self): mock_dd_transport = mock.Mock() - exporter = DatadogTaceExporter(options=Options(), - transport=MockTransport) + exporter = DatadogTraceExporter(options=Options(), + transport=MockTransport) exporter._dd_transport = mock_dd_transport exporter.export({}) self.assertTrue(exporter.transport.export_called) @mock.patch('opencensus.ext.datadog.traces.' - 'DatadogTaceExporter.translate_to_datadog', + 'DatadogTraceExporter.translate_to_datadog', return_value=None) def test_emit(self, mr_mock): @@ -122,8 +124,9 @@ def test_emit(self, mr_mock): ] mock_dd_transport = mock.Mock() - exporter = DatadogTaceExporter(options=Options(service="dd-unit-test"), - transport=MockTransport) + exporter = DatadogTraceExporter( + options=Options(service="dd-unit-test"), + transport=MockTransport) exporter._dd_transport = mock_dd_transport exporter.emit(span_datas) @@ -219,7 +222,7 @@ def test_translate_to_datadog(self): mock_dd_transport = mock.Mock() opts = Options(service="dd-unit-test") tran = MockTransport - exporter = DatadogTaceExporter(options=opts, transport=tran) + exporter = DatadogTraceExporter(options=opts, transport=tran) exporter._dd_transport = mock_dd_transport trace = { 'spans': [{ diff --git a/contrib/opencensus-ext-datadog/tests/transport_test.py b/contrib/opencensus-ext-datadog/tests/transport_test.py index 51838bcaa..56cd5b318 100644 --- a/contrib/opencensus-ext-datadog/tests/transport_test.py +++ b/contrib/opencensus-ext-datadog/tests/transport_test.py @@ -1,4 +1,5 @@ import unittest + import mock from opencensus.ext.datadog.transport import DDTransport From ede8d08b390778e8ac2c7bf3dfa8d03a7f71fabf Mon Sep 17 00:00:00 2001 From: Nathan Button Date: Mon, 30 Sep 2019 18:14:18 -0700 Subject: [PATCH 4/5] Setup version info --- contrib/opencensus-ext-datadog/CHANGELOG.md | 4 ++++ contrib/opencensus-ext-datadog/setup.cfg | 2 ++ contrib/opencensus-ext-datadog/setup.py | 6 +++++- contrib/opencensus-ext-datadog/version.py | 15 +++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 contrib/opencensus-ext-datadog/CHANGELOG.md create mode 100644 contrib/opencensus-ext-datadog/setup.cfg create mode 100644 contrib/opencensus-ext-datadog/version.py diff --git a/contrib/opencensus-ext-datadog/CHANGELOG.md b/contrib/opencensus-ext-datadog/CHANGELOG.md new file mode 100644 index 000000000..b2705a57e --- /dev/null +++ b/contrib/opencensus-ext-datadog/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## Unreleased +- Initial version. diff --git a/contrib/opencensus-ext-datadog/setup.cfg b/contrib/opencensus-ext-datadog/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/contrib/opencensus-ext-datadog/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/contrib/opencensus-ext-datadog/setup.py b/contrib/opencensus-ext-datadog/setup.py index 8ac96d1c7..48aa39367 100644 --- a/contrib/opencensus-ext-datadog/setup.py +++ b/contrib/opencensus-ext-datadog/setup.py @@ -40,10 +40,14 @@ install_requires=[ 'bitarray >= 1.0.1, < 2.0.0', 'opencensus >= 0.8.dev0, < 1.0.0', + 'requests >= 2.19.0', ], extras_require={}, license='Apache-2.0', - packages=find_packages(exclude=('examples', 'tests',)), + packages=find_packages(exclude=( + 'examples', + 'tests', + )), namespace_packages=[], url='https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-datadog', # noqa: E501 zip_safe=False, diff --git a/contrib/opencensus-ext-datadog/version.py b/contrib/opencensus-ext-datadog/version.py new file mode 100644 index 000000000..f3a64a892 --- /dev/null +++ b/contrib/opencensus-ext-datadog/version.py @@ -0,0 +1,15 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = '0.1.dev0' From 91513f3669b378922b8fa01f49fb48f55db445b1 Mon Sep 17 00:00:00 2001 From: Nathan Button Date: Mon, 30 Sep 2019 18:32:39 -0700 Subject: [PATCH 5/5] define dd exporter in tox.ini --- .../opencensus/ext/datadog/traces.py | 47 +++++++++---------- tox.ini | 1 + 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py index 711c2c92a..ebeee9b84 100644 --- a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py @@ -18,6 +18,7 @@ import bitarray from opencensus.common.transports import sync +from opencensus.common.utils import ISO_DATETIME_REGEX from opencensus.ext.datadog.transport import DDTransport from opencensus.trace import base_exporter from opencensus.trace import span_data @@ -159,14 +160,16 @@ def translate_to_datadog(self, trace): 'resource': span.get("displayName").get("value"), } - start_time = datetime.strptime(span.get('startTime'), TIME_FMT) + start_time = datetime.strptime(span.get('startTime'), + ISO_DATETIME_REGEX) # The start time of the request in nanoseconds from the unix epoch. epoch = datetime.utcfromtimestamp(0) dd_span["start"] = int((start_time - epoch).total_seconds() * 1000.0 * 1000.0 * 1000.0) - end_time = datetime.strptime(span.get('endTime'), TIME_FMT) + end_time = datetime.strptime(span.get('endTime'), + ISO_DATETIME_REGEX) duration_td = end_time - start_time # The duration of the request in nanoseconds. @@ -186,8 +189,7 @@ def translate_to_datadog(self, trace): # opencensus.trace.span.SpanKind dd_span['type'] = to_dd_type(span.get("kind")) dd_span["error"] = 0 - if int(code.get("status") / 100) == 4 or int( - code.get("status") / 100) == 5: + if 4 <= code.get("status") // 100 <= 5: dd_span["error"] = 1 meta["error.type"] = code.get("message") @@ -298,77 +300,74 @@ def convert_id(str_id): return id_int -# 2019-09-19T14:05:15.808570Z -TIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" - # https://opencensus.io/tracing/span/status/ STATUS_CODES = { 0: { "message": "OK", - "status": int(200) + "status": 200 }, 1: { "message": "CANCELLED", - "status": int(499) + "status": 499 }, 2: { "message": "UNKNOWN", - "status": int(500) + "status": 500 }, 3: { "message": "INVALID_ARGUMENT", - "status": int(400) + "status": 400 }, 4: { "message": "DEADLINE_EXCEEDED", - "status": int(504) + "status": 504 }, 5: { "message": "NOT_FOUND", - "status": int(404) + "status": 404 }, 6: { "message": "ALREADY_EXISTS", - "status": int(409) + "status": 409 }, 7: { "message": "PERMISSION_DENIED", - "status": int(403) + "status": 403 }, 8: { "message": "RESOURCE_EXHAUSTED", - "status": int(429) + "status": 429 }, 9: { "message": "FAILED_PRECONDITION", - "status": int(400) + "status": 400 }, 10: { "message": "ABORTED", - "status": int(409) + "status": 409 }, 11: { "message": "OUT_OF_RANGE", - "status": int(400) + "status": 400 }, 12: { "message": "UNIMPLEMENTED", - "status": int(502) + "status": 502 }, 13: { "message": "INTERNAL", - "status": int(500) + "status": 500 }, 14: { "message": "UNAVAILABLE", - "status": int(503) + "status": 503 }, 15: { "message": "DATA_LOSS", - "status": int(501) + "status": 501 }, 16: { "message": "UNAUTHENTICATED", - "status": int(401) + "status": 401 }, } diff --git a/tox.ini b/tox.ini index 845972b34..74a05863f 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e contrib/opencensus-correlation py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e . py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-azure + py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-datadog py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-dbapi py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-django py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-flask