From 68850b3a4e4ed8cf39b2b2ad06aa58800846407e Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 14:50:08 -0500 Subject: [PATCH 01/54] TODO: AMEND: add W3C style, but it doesn't do anything --- src/datadog/propagation_style.cpp | 6 +++++- src/datadog/propagation_style.h | 2 ++ src/datadog/trace_segment.cpp | 6 +++++- src/datadog/tracer.cpp | 10 ++++++++++ src/datadog/tracer_config.cpp | 5 +++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/datadog/propagation_style.cpp b/src/datadog/propagation_style.cpp index 146c6cd1..ca4beb8b 100644 --- a/src/datadog/propagation_style.cpp +++ b/src/datadog/propagation_style.cpp @@ -8,11 +8,15 @@ namespace datadog { namespace tracing { nlohmann::json to_json(PropagationStyle style) { + // Note: Make sure that these strings are consistent (modulo case) with + // `parse_propagation_styles` in `tracer_config.cpp`. switch (style) { case PropagationStyle::DATADOG: - return "datadog"; + return "Datadog"; case PropagationStyle::B3: return "B3"; + case PropagationStyle::W3C: + return "tracecontext"; // for compatibility with OpenTelemetry default: assert(style == PropagationStyle::NONE); return "none"; diff --git a/src/datadog/propagation_style.h b/src/datadog/propagation_style.h index 66d29786..ed32efcc 100644 --- a/src/datadog/propagation_style.h +++ b/src/datadog/propagation_style.h @@ -17,6 +17,8 @@ enum class PropagationStyle { DATADOG, // B3 multi-header style, e.g. X-B3-TraceID B3, + // W3C headers style, i.e. traceparent and tracestate + W3C, // The absence of propagation. If this is the only style set, then // propagation is disabled in the relevant direction (extraction or // injection). diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index ac70d96e..878a3147 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -261,7 +261,11 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); break; - case PropagationStyle::NONE: + case PropagationStyle::W3C: + // TODO + break; + default: + assert(style == PropagationStyle::NONE); break; } } diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 81f3bf25..2fbb365d 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -148,6 +148,13 @@ Expected extract_b3(const DictReader& headers) { return result; } +Expected extract_w3c(const DictReader& headers) { + ExtractedData result; + // TODO + (void)headers; + return result; +} + void log_startup_message(Logger& logger, StringView tracer_version_string, const Collector& collector, const SpanDefaults& defaults, @@ -256,6 +263,9 @@ Expected Tracer::extract_span(const DictReader& reader, case PropagationStyle::B3: extract = &extract_b3; break; + case PropagationStyle::W3C: + extract = &extract_w3c; + break; default: assert(style == PropagationStyle::NONE); extracted_data = ExtractedData{}; diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 814bbef3..6bf95b1a 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -94,10 +94,15 @@ Expected> parse_propagation_styles( for (const StringView &item : parse_list(input)) { auto token = std::string(item); to_lower(token); + // Note: Make sure that these strings are consistent (modulo case) with + // `to_json(PropagationStyle)` in `propagation_style.cpp`. if (token == "datadog") { styles.push_back(PropagationStyle::DATADOG); } else if (token == "b3" || token == "b3multi") { styles.push_back(PropagationStyle::B3); + } else if (token == + "tracecontext") { // for compatibility with OpenTelemetry + styles.push_back(PropagationStyle::W3C); } else if (token == "none") { styles.push_back(PropagationStyle::NONE); } else { From 8238cd5e39ceeb93088119f83b8ed35faf626fd4 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 18:26:04 -0500 Subject: [PATCH 02/54] TODO: AMEND: untested traceparent extraction --- BUILD.bazel | 4 ++ CMakeLists.txt | 4 ++ src/datadog/extracted_data.cpp | 1 + src/datadog/extracted_data.h | 28 ++++++++ src/datadog/tags.cpp | 2 + src/datadog/tags.h | 2 + src/datadog/tracer.cpp | 38 +++++------ src/datadog/w3c_propagation.cpp | 110 ++++++++++++++++++++++++++++++++ src/datadog/w3c_propagation.h | 23 +++++++ 9 files changed, 191 insertions(+), 21 deletions(-) create mode 100644 src/datadog/extracted_data.cpp create mode 100644 src/datadog/extracted_data.h create mode 100644 src/datadog/w3c_propagation.cpp create mode 100644 src/datadog/w3c_propagation.h diff --git a/BUILD.bazel b/BUILD.bazel index 088228a5..290cb07e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,6 +16,7 @@ cc_library( "src/datadog/error.cpp", "src/datadog/event_scheduler.cpp", "src/datadog/expected.cpp", + "src/datadog/extracted_data.cpp", "src/datadog/glob.cpp", "src/datadog/http_client.cpp", "src/datadog/id_generator.cpp", @@ -49,6 +50,7 @@ cc_library( "src/datadog/trace_sampler.cpp", "src/datadog/trace_segment.cpp", "src/datadog/version.cpp", + "src/datadog/w3c_propagation.cpp", ], hdrs = [ "src/datadog/cerr_logger.h", @@ -65,6 +67,7 @@ cc_library( "src/datadog/error.h", "src/datadog/event_scheduler.h", "src/datadog/expected.h", + "src/datadog/extracted_data.h", "src/datadog/glob.h", "src/datadog/http_client.h", "src/datadog/id_generator.h", @@ -100,6 +103,7 @@ cc_library( "src/datadog/trace_sampler.h", "src/datadog/trace_segment.h", "src/datadog/version.h", + "src/datadog/w3c_propagation.h", ], copts = [ "-Wall", diff --git a/CMakeLists.txt b/CMakeLists.txt index b6c842c7..cb42dbe4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/error.cpp src/datadog/event_scheduler.cpp src/datadog/expected.cpp + src/datadog/extracted_data.cpp src/datadog/glob.cpp src/datadog/http_client.cpp src/datadog/id_generator.cpp @@ -103,6 +104,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/trace_sampler.cpp src/datadog/trace_segment.cpp src/datadog/version.cpp + src/datadog/w3c_propagation.cpp ) # This library's public headers are just its source headers. @@ -125,6 +127,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/error.h src/datadog/event_scheduler.h src/datadog/expected.h + src/datadog/extracted_data.h src/datadog/glob.h src/datadog/http_client.h src/datadog/id_generator.h @@ -160,6 +163,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/trace_sampler.h src/datadog/trace_segment.h src/datadog/version.h + src/datadog/w3c_propagation.h ) add_dependencies(dd_trace_cpp curl) diff --git a/src/datadog/extracted_data.cpp b/src/datadog/extracted_data.cpp new file mode 100644 index 00000000..9d02ba2e --- /dev/null +++ b/src/datadog/extracted_data.cpp @@ -0,0 +1 @@ +#include "extracted_data.h" diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h new file mode 100644 index 00000000..96fb468f --- /dev/null +++ b/src/datadog/extracted_data.h @@ -0,0 +1,28 @@ +#pragma once + +// TODO: document + +#include +#include + +#include "optional.h" + +namespace datadog { +namespace tracing { + +struct ExtractedData { + Optional trace_id; + Optional parent_id; + Optional origin; + Optional trace_tags; + Optional sampling_priority; + // TODO: document + Optional full_w3c_trace_id_hex; + // TODO: document + Optional additional_w3c_tracestate; + // TODO: document + Optional additional_datadog_w3c_tracestate; +}; + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/tags.cpp b/src/datadog/tags.cpp index 4a3ebdd0..84b91202 100644 --- a/src/datadog/tags.cpp +++ b/src/datadog/tags.cpp @@ -28,6 +28,8 @@ const std::string agent_sample_rate = "_dd.agent_psr"; const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism"; const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate"; const std::string span_sampling_limit = "_dd.span_sampling.max_per_second"; +const std::string w3c_extraction_error = "_dd.w3c_extraction_error"; +const std::string w3c_injection_error = "_dd.w3c_injection_error"; } // namespace internal diff --git a/src/datadog/tags.h b/src/datadog/tags.h index ad79c866..b1d77c58 100644 --- a/src/datadog/tags.h +++ b/src/datadog/tags.h @@ -32,6 +32,8 @@ extern const std::string agent_sample_rate; extern const std::string span_sampling_mechanism; extern const std::string span_sampling_rule_rate; extern const std::string span_sampling_limit; +extern const std::string w3c_extraction_error; +extern const std::string w3c_injection_error; } // namespace internal // Return whether the specified `tag_name` is reserved for use internal to this diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 2fbb365d..644aa1fb 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -6,6 +6,7 @@ #include "datadog_agent.h" #include "dict_reader.h" #include "environment.h" +#include "extracted_data.h" #include "json.hpp" #include "logger.h" #include "net_util.h" @@ -19,6 +20,7 @@ #include "trace_sampler.h" #include "trace_segment.h" #include "version.h" +#include "w3c_propagation.h" namespace datadog { namespace tracing { @@ -50,15 +52,9 @@ Expected> extract_id_header(const DictReader& headers, return *result; } -struct ExtractedData { - Optional trace_id; - Optional parent_id; - Optional origin; - Optional trace_tags; - Optional sampling_priority; -}; - -Expected extract_datadog(const DictReader& headers) { +Expected extract_datadog( + const DictReader& headers, + std::unordered_map& /*span_tags*/) { ExtractedData result; auto trace_id = @@ -103,7 +99,9 @@ Expected extract_datadog(const DictReader& headers) { return result; } -Expected extract_b3(const DictReader& headers) { +Expected extract_b3( + const DictReader& headers, + std::unordered_map& /*span_tags*/) { ExtractedData result; auto trace_id = extract_id_header(headers, "x-b3-traceid", "trace", "B3", 16); @@ -148,13 +146,6 @@ Expected extract_b3(const DictReader& headers) { return result; } -Expected extract_w3c(const DictReader& headers) { - ExtractedData result; - // TODO - (void)headers; - return result; -} - void log_startup_message(Logger& logger, StringView tracer_version_string, const Collector& collector, const SpanDefaults& defaults, @@ -251,6 +242,7 @@ Expected Tracer::extract_span(const DictReader& reader, const SpanConfig& config) { assert(!extraction_styles_.empty()); + auto span_data = std::make_unique(); ExtractedData extracted_data; for (const auto style : extraction_styles_) { @@ -271,7 +263,7 @@ Expected Tracer::extract_span(const DictReader& reader, extracted_data = ExtractedData{}; continue; } - auto data = extract(reader); + auto data = extract(reader, span_data->tags); if (auto* error = data.if_error()) { return std::move(*error); } @@ -284,8 +276,13 @@ Expected Tracer::extract_span(const DictReader& reader, } } - auto& [trace_id, parent_id, origin, trace_tags, sampling_priority] = - extracted_data; + auto& [trace_id, parent_id, origin, trace_tags, sampling_priority, + full_w3c_trace_id_hex, additional_w3c_tracestate, + additional_datadog_w3c_tracestate] = extracted_data; + + (void)full_w3c_trace_id_hex; // TODO + (void)additional_w3c_tracestate; // TODO + (void)additional_datadog_w3c_tracestate; // TODO // Some information might be missing. // Here are the combinations considered: @@ -332,7 +329,6 @@ Expected Tracer::extract_span(const DictReader& reader, assert(parent_id); assert(trace_id); - auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); span_data->span_id = generator_(); span_data->trace_id = *trace_id; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp new file mode 100644 index 00000000..88d9c269 --- /dev/null +++ b/src/datadog/w3c_propagation.cpp @@ -0,0 +1,110 @@ +#include "w3c_propagation.h" + +#include +#include +#include +#include + +#include "dict_reader.h" +#include "parse_util.h" +#include "tags.h" + +namespace datadog { +namespace tracing { +namespace { + +// TODO: document +Optional extract_traceparent(ExtractedData& result, + const DictReader& headers) { + const auto maybe_traceparent = headers.lookup("traceparent"); + if (!maybe_traceparent) { + return nullopt; + } + + const auto traceparent = strip(*maybe_traceparent); + + // Note that leading and trailing whitespace was already removed above. + // Note that the "zero'th" match group is the entire match. + static const auto& pattern = + "([0-9a-f]{2})" // hex version number (match group 1) + "-" + "([0-9a-f]{16}([0-9a-f]{16}))" // hex trace ID (match groups 2 and 3) + "-" + "([0-9a-f]{16})" // hex parent span ID (match group 4) + "-" + "([0-9a-f]{2})" // hex "trace-flags" (match group 5) + "(?:$|-.*)"; // either the end, or a hyphen preceding further fields + thread_local const std::regex regex{pattern}; + + std::match_results match; + if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { + return "malformed_traceparent"; + } + + assert(match.ready()); + assert(match.size() == 5 + 1); + + const auto to_string_view = [](const auto& submatch) { + assert(submatch.first <= submatch.second); + return StringView{submatch.first, + std::size_t(submatch.second - submatch.first)}; + }; + + result.full_w3c_trace_id_hex = std::string{to_string_view(match[2])}; + if (result.full_w3c_trace_id_hex->find_first_not_of('0') == + std::string::npos) { + return "trace_id_zero"; + } + + result.trace_id = *parse_uint64(to_string_view(match[3]), 16); + + result.parent_id = *parse_uint64(to_string_view(match[4]), 16); + if (*result.parent_id == 0) { + return "parent_id_zero"; + } + + const auto flags = *parse_uint64(to_string_view(match[5]), 16); + result.sampling_priority = int(flags & 1); + + return nullopt; +} + +// TODO: document +Optional extract_tracestate(ExtractedData& result, + const DictReader& headers) { + // TODO + (void)result; + (void)headers; + return nullopt; +} + +} // namespace + +Expected extract_w3c( + const DictReader& headers, + std::unordered_map& span_tags) { + ExtractedData result; + + if (auto error_tag_value = extract_traceparent(result, headers)) { + span_tags[tags::internal::w3c_extraction_error] = + std::move(*error_tag_value); + return ExtractedData{}; + } + + // If we didn't get a trace ID from traceparent, don't bother with + // tracestate. + if (!result.trace_id) { + return result; + } + + if (auto error_tag_value = extract_tracestate(result, headers)) { + span_tags[tags::internal::w3c_extraction_error] = + std::move(*error_tag_value); + // Carry on with whatever data was extracted from traceparent. + } + + return result; +} + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h new file mode 100644 index 00000000..6be7d6f8 --- /dev/null +++ b/src/datadog/w3c_propagation.h @@ -0,0 +1,23 @@ +#pragma once + +// TODO: document + +#include +#include + +#include "expected.h" +#include "extracted_data.h" +#include "optional.h" + +namespace datadog { +namespace tracing { + +class DictReader; + +// TODO: document +Expected extract_w3c( + const DictReader& headers, + std::unordered_map& span_tags); + +} // namespace tracing +} // namespace datadog From 1a06e865ea1ab8d9d647426bcdb55ec48e1cb1f1 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 19:46:26 -0500 Subject: [PATCH 03/54] TODO: AMEND: it's already thread-safe --- src/datadog/w3c_propagation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 88d9c269..8e517c8c 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -34,7 +34,7 @@ Optional extract_traceparent(ExtractedData& result, "-" "([0-9a-f]{2})" // hex "trace-flags" (match group 5) "(?:$|-.*)"; // either the end, or a hyphen preceding further fields - thread_local const std::regex regex{pattern}; + static const std::regex regex{pattern}; std::match_results match; if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { From 122fc3d148ee600b51e130859e3b52ef2653eebb Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 19:50:34 -0500 Subject: [PATCH 04/54] add a convenience script that does some of the CI checks --- bin/check | 13 +++++++++++++ bin/format | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 bin/check diff --git a/bin/check b/bin/check new file mode 100755 index 00000000..2d5b4fde --- /dev/null +++ b/bin/check @@ -0,0 +1,13 @@ +#!/bin/sh + +# Run some of the checks that are performed by CircleCI. +# This is convenient to run before committing. + +set -e + +# Go to the repository root directory. +cd "$(dirname "$0")"/.. + +bin/format --dry-run -Werror +bin/test +bin/bazel-build diff --git a/bin/format b/bin/format index 119f7536..5efa9fd9 100755 --- a/bin/format +++ b/bin/format @@ -9,7 +9,7 @@ cd "$(dirname "$0")"/.. # occasionally bumps the required version, reformatting everything. version=14 formatter=clang-format-$version -formatter_options='--style=file -i' +formatter_options="--style=file -i $@" find_sources() { find src/ example/ test/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" From fea21152a5a7ca5f55ca71017848a3f8f6c66d70 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 20:00:04 -0500 Subject: [PATCH 05/54] TODO: AMEND: whoops --- src/datadog/w3c_propagation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 8e517c8c..2b0dfbc9 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -37,7 +37,7 @@ Optional extract_traceparent(ExtractedData& result, static const std::regex regex{pattern}; std::match_results match; - if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { + if (!std::regex_match(traceparent.begin(), traceparent.end(), match, regex)) { return "malformed_traceparent"; } From aaea195403aa8c0add7c256275cf693d9771a320 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 18:50:04 -0500 Subject: [PATCH 06/54] TODO: AMEND: extract traceparent --- src/datadog/span_data.cpp | 4 +- src/datadog/trace_segment.cpp | 9 +- src/datadog/trace_segment.h | 6 ++ src/datadog/tracer.cpp | 11 ++- src/datadog/w3c_propagation.cpp | 6 +- test/test.cpp | 2 +- test/test.h | 7 +- test/test_tracer.cpp | 149 +++++++++++++++++++++++++++++++- 8 files changed, 180 insertions(+), 14 deletions(-) diff --git a/src/datadog/span_data.cpp b/src/datadog/span_data.cpp index bf83cc32..fea93cad 100644 --- a/src/datadog/span_data.cpp +++ b/src/datadog/span_data.cpp @@ -38,7 +38,9 @@ void SpanData::apply_config(const SpanDefaults& defaults, service = config.service.value_or(defaults.service); name = config.name.value_or(defaults.name); - tags = defaults.tags; + for (const auto& item : defaults.tags) { + tags.insert(item); + } std::string environment = config.environment.value_or(defaults.environment); if (!environment.empty()) { tags.insert_or_assign(tags::environment, environment); diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 878a3147..254fdba6 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -50,6 +50,9 @@ TraceSegment::TraceSegment( std::size_t tags_header_max_size, std::unordered_map trace_tags, Optional sampling_decision, + Optional full_w3c_trace_id_hex, + Optional additional_w3c_tracestate, + Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root) : logger_(logger), collector_(collector), @@ -62,7 +65,11 @@ TraceSegment::TraceSegment( tags_header_max_size_(tags_header_max_size), trace_tags_(std::move(trace_tags)), num_finished_spans_(0), - sampling_decision_(std::move(sampling_decision)) { + sampling_decision_(std::move(sampling_decision)), + full_w3c_trace_id_hex_(std::move(full_w3c_trace_id_hex)), + additional_w3c_tracestate_(std::move(additional_w3c_tracestate)), + additional_datadog_w3c_tracestate_( + std::move(additional_datadog_w3c_tracestate)) { assert(logger_); assert(collector_); assert(trace_sampler_); diff --git a/src/datadog/trace_segment.h b/src/datadog/trace_segment.h index 0256e405..772e9c36 100644 --- a/src/datadog/trace_segment.h +++ b/src/datadog/trace_segment.h @@ -67,6 +67,9 @@ class TraceSegment { std::vector> spans_; std::size_t num_finished_spans_; Optional sampling_decision_; + Optional full_w3c_trace_id_hex_; + Optional additional_w3c_tracestate_; + Optional additional_datadog_w3c_tracestate_; bool awaiting_delegated_sampling_decision_ = false; public: @@ -80,6 +83,9 @@ class TraceSegment { Optional origin, std::size_t tags_header_max_size, std::unordered_map trace_tags, Optional sampling_decision, + Optional full_w3c_trace_id_hex, + Optional additional_w3c_tracestate, + Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root); const SpanDefaults& defaults() const; diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 644aa1fb..7dae2697 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -229,7 +229,9 @@ Span Tracer::create_span(const SpanConfig& config) { logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, std::unordered_map{} /* trace_tags */, - nullopt /* sampling_decision */, std::move(span_data)); + nullopt /* sampling_decision */, nullopt /* full_w3c_trace_id_hex */, + nullopt /* additional_w3c_tracestate */, + nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; } @@ -280,10 +282,6 @@ Expected Tracer::extract_span(const DictReader& reader, full_w3c_trace_id_hex, additional_w3c_tracestate, additional_datadog_w3c_tracestate] = extracted_data; - (void)full_w3c_trace_id_hex; // TODO - (void)additional_w3c_tracestate; // TODO - (void)additional_datadog_w3c_tracestate; // TODO - // Some information might be missing. // Here are the combinations considered: // @@ -365,7 +363,8 @@ Expected Tracer::extract_span(const DictReader& reader, logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, std::move(origin), tags_header_max_size_, std::move(decoded_trace_tags), std::move(sampling_decision), - std::move(span_data)); + std::move(full_w3c_trace_id_hex), std::move(additional_w3c_tracestate), + std::move(additional_datadog_w3c_tracestate), std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; } diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 2b0dfbc9..7b573460 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -24,7 +24,7 @@ Optional extract_traceparent(ExtractedData& result, const auto traceparent = strip(*maybe_traceparent); // Note that leading and trailing whitespace was already removed above. - // Note that the "zero'th" match group is the entire match. + // Note that the match group 0 is the entire match. static const auto& pattern = "([0-9a-f]{2})" // hex version number (match group 1) "-" @@ -50,6 +50,10 @@ Optional extract_traceparent(ExtractedData& result, std::size_t(submatch.second - submatch.first)}; }; + if (to_string_view(match[1]) == "ff") { + return "invalid_version"; + } + result.full_w3c_trace_id_hex = std::string{to_string_view(match[2])}; if (result.full_w3c_trace_id_hex->find_first_not_of('0') == std::string::npos) { diff --git a/test/test.cpp b/test/test.cpp index 402ae6cc..ca57ed79 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -11,7 +11,7 @@ std::ostream& operator<<( std::ostream& operator<<( std::ostream& stream, const datadog::tracing::Optional& item) { - return stream << item.value_or(""); + return stream << item.value_or(""); } } // namespace std diff --git a/test/test.h b/test/test.h index a0b16615..e9a8664a 100644 --- a/test/test.h +++ b/test/test.h @@ -11,7 +11,9 @@ #include #include +#include #include +#include #include #include "catch.hpp" @@ -21,9 +23,8 @@ namespace std { std::ostream& operator<<(std::ostream& stream, const std::pair& item); -std::ostream& operator<<( - std::ostream& stream, - const datadog::tracing::Optional& item); +std::ostream& operator<<(std::ostream& stream, + const std::optional& maybe); } // namespace std diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 9f0ebdc7..981c2b35 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -239,7 +240,8 @@ TEST_CASE("tracer span defaults") { TEST_CASE("span extraction") { TracerConfig config; config.defaults.service = "testsvc"; - config.collector = std::make_shared(); + const auto collector = std::make_shared(); + config.collector = collector; config.logger = std::make_shared(); SECTION( @@ -504,6 +506,151 @@ TEST_CASE("span extraction") { REQUIRE(result.error().code == Error::NO_SPAN_TO_EXTRACT); } + SECTION("W3C traceparent extraction") { + const std::unordered_map datadog_headers{ + {"x-datadog-trace-id", "18"}, + {"x-datadog-parent-id", "23"}, + {"x-datadog-sampling-priority", "-1"}, + }; + + struct TestCase { + int line; + std::string name; + Optional traceparent; + Optional expected_error_tag_value = {}; + Optional expected_full_trace_id = {}; + Optional expected_trace_id = {}; + Optional expected_parent_id = {}; + Optional expected_sampling_priority = {}; + }; + + // clang-format off + auto test_case = GENERATE(values({ + // https://www.w3.org/TR/trace-context/#examples-of-http-traceparent-headers + {__LINE__, "valid: w3.org example 1", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 1}, // expected_sampling_priority + + {__LINE__, "valid: w3.org example 2", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "valid: future version", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "valid: future version with extra fields", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00-af-delta", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "no traceparent", + nullopt}, // traceparent + + {__LINE__, "invalid: not enough fields", + "06-4bf92f3577b34da6a3ce929d0e0e4736", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: missing hyphen", + "064bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: extra data not preceded by hyphen", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00af-delta", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: version", + "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + "invalid_version"}, // expected_error_tag_value + + {__LINE__, "invalid: trace ID zero", + "00-00000000000000000000000000000000-00f067aa0ba902b7-00", // traceparent + "trace_id_zero"}, // expected_error_tag_value + + {__LINE__, "invalid: parent ID zero", + "00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-00", // traceparent + "parent_id_zero"}, // expected_error_tag_value + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + + config.extraction_styles = {PropagationStyle::W3C, + PropagationStyle::DATADOG}; + auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + Tracer tracer{*finalized_config}; + + auto headers = datadog_headers; + if (test_case.traceparent) { + headers["traceparent"] = *test_case.traceparent; + } + MockDictReader reader{headers}; + + // We can't `span->lookup(tags::internal::w3c_extraction_error)`, because + // that tag is internal and will not be returned by `lookup`. Instead, we + // finish (destroy) the span to send it to a collector, and then inspect the + // `SpanData` at the collector. + Optional decision; + { + auto span = tracer.extract_span(reader); + REQUIRE(span); + decision = span->trace_segment().sampling_decision(); + } + + REQUIRE(collector->span_count() == 1); + const auto& span_data = collector->first_span(); + + if (test_case.expected_error_tag_value) { + const auto error_found = + span_data.tags.find(tags::internal::w3c_extraction_error); + REQUIRE(error_found != span_data.tags.end()); + REQUIRE(error_found->second == *test_case.expected_error_tag_value); + // Extraction would have fallen back to the next configured style (Datadog + // -- see `config.extraction_styles`, above), and so the span's properties + // should match `datadog_headers`, above. + REQUIRE(span_data.trace_id == 18); + REQUIRE(span_data.parent_id == 23); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == -1); + } else if (!test_case.traceparent) { + // There was no error extracting W3C context, but there was none to + // extract. + // Extraction would have fallen back to the next configured style (Datadog + // -- see `config.extraction_styles`, above), and so the span's properties + // should match `datadog_headers`, above. + REQUIRE(span_data.trace_id == 18); + REQUIRE(span_data.parent_id == 23); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == -1); + } else { + // W3C context was successfully extracted from traceparent header. + REQUIRE(span_data.trace_id == *test_case.expected_trace_id); + REQUIRE(span_data.parent_id == *test_case.expected_parent_id); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == *test_case.expected_sampling_priority); + } + } + SECTION("x-datadog-tags") { auto finalized_config = finalize_config(config); REQUIRE(finalized_config); From b34231e8e6ba1a2237057b4c8147695b4c42310c Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 19:03:26 -0500 Subject: [PATCH 07/54] TODO: AMEND: add some documentation --- src/datadog/w3c_propagation.cpp | 4 +++- src/datadog/w3c_propagation.h | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 7b573460..a5f39dd1 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -13,7 +13,9 @@ namespace datadog { namespace tracing { namespace { -// TODO: document +// Populate the specified `result` with data extracted from the "traceparent" +// entry of the specified `headers`. Return `nullopt` on success. Return a value +// for the `tags::internal::w3c_extraction_error` tag if an error occurs. Optional extract_traceparent(ExtractedData& result, const DictReader& headers) { const auto maybe_traceparent = headers.lookup("traceparent"); diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 6be7d6f8..36486a1e 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -14,7 +14,11 @@ namespace tracing { class DictReader; -// TODO: document +// Return `ExtractedData` deduced from the "traceparent" and "tracestate" +// entries of the specified `headers`. If an error occurs, set a value for the +// `tags::internal::w3c_extraction_error` tag in the specified `span_tags`. +// `extract_w3c` will not return an error; instead, it returns an empty +// `ExtractedData` when extraction fails. Expected extract_w3c( const DictReader& headers, std::unordered_map& span_tags); From d0005cdec3b945db1cdf79892a11504868816890 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 20:26:56 -0500 Subject: [PATCH 08/54] TODO: AMEND: inject traceparent --- BUILD.bazel | 2 + CMakeLists.txt | 2 + src/datadog/hex.cpp | 1 + src/datadog/hex.h | 31 +++++++++++++ src/datadog/trace_segment.cpp | 26 +++-------- src/datadog/w3c_propagation.cpp | 30 +++++++++++++ src/datadog/w3c_propagation.h | 9 ++++ test/test_span.cpp | 79 +++++++++++++++++++++++++++++++++ 8 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 src/datadog/hex.cpp create mode 100644 src/datadog/hex.h diff --git a/BUILD.bazel b/BUILD.bazel index 290cb07e..d78fd98a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -18,6 +18,7 @@ cc_library( "src/datadog/expected.cpp", "src/datadog/extracted_data.cpp", "src/datadog/glob.cpp", + "src/datadog/hex.cpp", "src/datadog/http_client.cpp", "src/datadog/id_generator.cpp", "src/datadog/limiter.cpp", @@ -69,6 +70,7 @@ cc_library( "src/datadog/expected.h", "src/datadog/extracted_data.h", "src/datadog/glob.h", + "src/datadog/hex.h", "src/datadog/http_client.h", "src/datadog/id_generator.h", "src/datadog/json.hpp", diff --git a/CMakeLists.txt b/CMakeLists.txt index cb42dbe4..9c93261e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/expected.cpp src/datadog/extracted_data.cpp src/datadog/glob.cpp + src/datadog/hex.cpp src/datadog/http_client.cpp src/datadog/id_generator.cpp src/datadog/limiter.cpp @@ -129,6 +130,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/expected.h src/datadog/extracted_data.h src/datadog/glob.h + src/datadog/hex.h src/datadog/http_client.h src/datadog/id_generator.h src/datadog/json_fwd.hpp diff --git a/src/datadog/hex.cpp b/src/datadog/hex.cpp new file mode 100644 index 00000000..fe81d19e --- /dev/null +++ b/src/datadog/hex.cpp @@ -0,0 +1 @@ +#include "hex.h" diff --git a/src/datadog/hex.h b/src/datadog/hex.h new file mode 100644 index 00000000..dec38a2e --- /dev/null +++ b/src/datadog/hex.h @@ -0,0 +1,31 @@ +#pragma once + +// This component provides a function, `hex`, for formatting an integral value +// in hexadecimal. + +#include +#include +#include +#include +#include + +namespace datadog { +namespace tracing { + +// Return the specified `value` formatted as a lower-case hexadecimal string +// without any leading zeroes. +template +std::string hex(Integer value) { + // 4 bits per hex digit char, and then +1 char for possible minus sign + char buffer[std::numeric_limits::digits / 4 + 1]; + + const int base = 16; + auto result = + std::to_chars(std::begin(buffer), std::end(buffer), value, base); + assert(result.ec == std::errc()); + + return std::string{std::begin(buffer), result.ptr}; +} + +} // namespace tracing +} // namespace datadog \ No newline at end of file diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 254fdba6..214960f0 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -1,10 +1,7 @@ #include "trace_segment.h" #include -#include -#include #include -#include #include #include @@ -12,6 +9,7 @@ #include "collector_response.h" #include "dict_writer.h" #include "error.h" +#include "hex.h" #include "logger.h" #include "optional.h" #include "span_data.h" @@ -19,25 +17,10 @@ #include "tag_propagation.h" #include "tags.h" #include "trace_sampler.h" +#include "w3c_propagation.h" namespace datadog { namespace tracing { -namespace { - -template -std::string hex(Integer value) { - // 4 bits per hex digit char, and then +1 char for possible minus sign - char buffer[std::numeric_limits::digits / 4 + 1]; - - const int base = 16; - auto result = - std::to_chars(std::begin(buffer), std::end(buffer), value, base); - assert(result.ec == std::errc()); - - return std::string{std::begin(buffer), result.ptr}; -} - -} // namespace TraceSegment::TraceSegment( const std::shared_ptr& logger, @@ -269,7 +252,10 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); break; case PropagationStyle::W3C: - // TODO + writer.set("traceparent", + encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, + span.span_id, sampling_priority)); + // TODO writer.set("tracestate", ...); break; default: assert(style == PropagationStyle::NONE); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index a5f39dd1..41e0113a 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -6,6 +6,7 @@ #include #include "dict_reader.h" +#include "hex.h" #include "parse_util.h" #include "tags.h" @@ -112,5 +113,34 @@ Expected extract_w3c( return result; } +std::string encode_traceparent( + std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, + std::uint64_t span_id, int sampling_priority) { + std::string result; + // version + result += "00-"; + + // trace ID + if (full_w3c_trace_id_hex) { + result += *full_w3c_trace_id_hex; + } else { + auto hexed = hex(trace_id); + result.append(16 + (16 - hexed.size()), '0'); // leading zeroes + result += hexed; + } + result += '-'; + + // span ID + auto hexed = hex(span_id); + result.append(16 - hexed.size(), '0'); // leading zeroes + result += hexed; + result += '-'; + + // flags + result += sampling_priority > 0 ? "01" : "00"; + + return result; +} + } // namespace tracing } // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 36486a1e..26e594dc 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -2,6 +2,7 @@ // TODO: document +#include #include #include @@ -23,5 +24,13 @@ Expected extract_w3c( const DictReader& headers, std::unordered_map& span_tags); +// Return a value for the "traceparent" header consisting of the specified +// `trace_id` or the optionally specified `full_w3c_trace_id_hex` as the trace +// ID, the specified `span_id` as the parent ID, and trace flags deduced from +// the specified `sampling_priority`. +std::string encode_traceparent( + std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, + std::uint64_t span_id, int sampling_priority); + } // namespace tracing } // namespace datadog diff --git a/test/test_span.cpp b/test/test_span.cpp index b976b2de..2acc13b3 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -3,6 +3,7 @@ // for propagation. #include +#include #include #include #include @@ -465,3 +466,81 @@ TEST_CASE("injection can be disabled using the \"none\" style") { const std::unordered_map empty; REQUIRE(writer.items == empty); } + +TEST_CASE("injecting W3C traceparent header") { + TracerConfig config; + config.defaults.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.injection_styles = {PropagationStyle::W3C}; + + SECTION("extracted from W3C traceparent") { + config.extraction_styles = {PropagationStyle::W3C}; + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + + // Override the tracer's ID generator to always return `expected_parent_id`. + constexpr std::uint64_t expected_parent_id = 0xcafebabe; + Tracer tracer{*finalized_config, [=]() { return expected_parent_id; }, + default_clock}; + + const std::unordered_map input_headers{ + // https://www.w3.org/TR/trace-context/#examples-of-http-traceparent-headers + {"traceparent", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + }; + const MockDictReader reader{input_headers}; + const auto maybe_span = tracer.extract_span(reader); + REQUIRE(maybe_span); + const auto& span = *maybe_span; + REQUIRE(span.id() == expected_parent_id); + + MockDictWriter writer; + span.inject(writer); + const auto& output_headers = writer.items; + const auto found = output_headers.find("traceparent"); + REQUIRE(found != output_headers.end()); + // The "00000000cafebabe" is the zero-padded `expected_parent_id`. + const StringView expected = + "00-4bf92f3577b34da6a3ce929d0e0e4736-00000000cafebabe-01"; + REQUIRE(found->second == expected); + } + + SECTION("not extracted from W3C traceparent") { + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + + // Override the tracer's ID generator to always return a fixed value. + // This will be both the trace ID and the parent (root) span ID. + constexpr std::uint64_t expected_id = 0xcafebabe; + Tracer tracer{*finalized_config, [=]() { return expected_id; }, + default_clock}; + + auto span = tracer.create_span(); + + // Let's test the effect sampling priority plays on the resulting + // traceparent, too. + struct TestCase { + int sampling_priority; + std::string expected_flags; + }; + const auto& [sampling_priority, expected_flags] = GENERATE( + values({{-1, "00"}, {0, "00"}, {1, "01"}, {2, "01"}})); + + CAPTURE(sampling_priority); + CAPTURE(expected_flags); + + span.trace_segment().override_sampling_priority(sampling_priority); + + MockDictWriter writer; + span.inject(writer); + const auto& output_headers = writer.items; + const auto found = output_headers.find("traceparent"); + REQUIRE(found != output_headers.end()); + // The "cafebabe"s come from `expected_id`. + const std::string expected = + "00-000000000000000000000000cafebabe-00000000cafebabe-" + + expected_flags; + REQUIRE(found->second == expected); + } +} From 5ce2166be3b9dd44ee9279f5eeed355c7988dede Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 21:44:38 -0500 Subject: [PATCH 09/54] TODO: AMEND: untested tracestate extraction --- src/datadog/extracted_data.h | 3 +- src/datadog/tracer.cpp | 47 +++++---- src/datadog/w3c_propagation.cpp | 176 +++++++++++++++++++++++++++++++- src/datadog/w3c_propagation.h | 3 +- 4 files changed, 201 insertions(+), 28 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index 96fb468f..bfdd2dea 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -4,6 +4,7 @@ #include #include +#include #include "optional.h" @@ -14,7 +15,7 @@ struct ExtractedData { Optional trace_id; Optional parent_id; Optional origin; - Optional trace_tags; + std::unordered_map trace_tags; Optional sampling_priority; // TODO: document Optional full_w3c_trace_id_hex; diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 7dae2697..79d27dd3 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -26,6 +26,26 @@ namespace datadog { namespace tracing { namespace { +// Decode the specified `trace_tags` and integrate them into the specified +// `result`. If an error occurs, add a `tags::internal::propagation_error` tag +// to the specified `span_tags` and log a diagnostic using the specified +// `logger`. +void handle_trace_tags(StringView trace_tags, ExtractedData& result, + std::unordered_map& span_tags, + Logger& logger) { + auto maybe_trace_tags = decode_tags(trace_tags); + if (auto* error = maybe_trace_tags.if_error()) { + logger.log_error(*error); + span_tags[tags::internal::propagation_error] = "decoding_error"; + } else { + for (const auto& [key, value] : *maybe_trace_tags) { + if (starts_with(key, "_dd.p.")) { + result.trace_tags.insert_or_assign(key, value); + } + } + } +} + Expected> extract_id_header(const DictReader& headers, StringView header, StringView header_kind, @@ -54,7 +74,7 @@ Expected> extract_id_header(const DictReader& headers, Expected extract_datadog( const DictReader& headers, - std::unordered_map& /*span_tags*/) { + std::unordered_map& span_tags, Logger& logger) { ExtractedData result; auto trace_id = @@ -93,7 +113,7 @@ Expected extract_datadog( auto trace_tags = headers.lookup("x-datadog-tags"); if (trace_tags) { - result.trace_tags = std::string(*trace_tags); + handle_trace_tags(*trace_tags, result, span_tags, logger); } return result; @@ -101,7 +121,7 @@ Expected extract_datadog( Expected extract_b3( const DictReader& headers, - std::unordered_map& /*span_tags*/) { + std::unordered_map& span_tags, Logger& logger) { ExtractedData result; auto trace_id = extract_id_header(headers, "x-b3-traceid", "trace", "B3", 16); @@ -140,7 +160,7 @@ Expected extract_b3( auto trace_tags = headers.lookup("x-datadog-tags"); if (trace_tags) { - result.trace_tags = std::string(*trace_tags); + handle_trace_tags(*trace_tags, result, span_tags, logger); } return result; @@ -265,7 +285,7 @@ Expected Tracer::extract_span(const DictReader& reader, extracted_data = ExtractedData{}; continue; } - auto data = extract(reader, span_data->tags); + auto data = extract(reader, span_data->tags, *logger_); if (auto* error = data.if_error()) { return std::move(*error); } @@ -343,26 +363,11 @@ Expected Tracer::extract_span(const DictReader& reader, sampling_decision = decision; } - std::unordered_map decoded_trace_tags; - if (trace_tags) { - auto maybe_trace_tags = decode_tags(*trace_tags); - if (auto* error = maybe_trace_tags.if_error()) { - logger_->log_error(*error); - span_data->tags[tags::internal::propagation_error] = "decoding_error"; - } else { - for (const auto& [key, value] : *maybe_trace_tags) { - if (starts_with(key, "_dd.p.")) { - decoded_trace_tags.insert_or_assign(key, value); - } - } - } - } - const auto span_data_ptr = span_data.get(); const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, std::move(origin), tags_header_max_size_, - std::move(decoded_trace_tags), std::move(sampling_decision), + std::move(trace_tags), std::move(sampling_decision), std::move(full_w3c_trace_id_hex), std::move(additional_w3c_tracestate), std::move(additional_datadog_w3c_tracestate), std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 41e0113a..2e017e68 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -76,20 +76,186 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } +// TODO: document +struct PartiallyParsedTracestate { + StringView datadog_value; + std::string other_entries; +}; + +// TODO: document +Optional parse_tracestate(StringView tracestate) { + Optional result; + + const char* const begin = tracestate.begin(); + const char* const end = tracestate.end(); + const char* pair_begin = begin; + for (;;) { + const char* const pair_end = std::find(pair_begin, end, ','); + // Note that since this `pair` is `strip`ped, `pair_begin` is not + // necessarily equal to `pair.begin()` (similarly for the ends). + const auto pair = strip(range(pair_begin, pair_end)); + if (pair.empty()) { + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + const auto kv_separator = std::find(pair.begin(), pair.end(), '='); + if (kv_separator == pair.end()) { + // This is an invalid entry because it contains a non-whitespace character + // but not a "=". + // Let's move on to the next entry. + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + const auto key = range(pair.begin(), kv_separator); + if (key != "dd") { + // On to the next. + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + // We found the "dd" entry. + result.emplace(); + result->datadog_value = range(kv_separator + 1, pair.end()); + // `result->other_entries` is whatever was before the "dd" entry and + // whatever is after the "dd" entry, but without a redundant comma in the + // middle. + if (pair_begin != begin) { + // There's a prefix + append(result->other_entries, range(begin, pair_begin - 1)); + if (pair_end != end) { + // and a suffix + append(result->other_entries, range(pair_end, end)); + } + } else if (pair_end != end) { + // There's just a suffix + append(result->other_entries, range(pair_end + 1, end)); + } + + return result; + } +} + +// TODO: document +Optional parse_datadog_tracestate(ExtractedData& result, + StringView datadog_value) { + const char* const begin = datadog_value.begin(); + const char* const end = datadog_value.end(); + const char* pair_begin = begin; + for (;;) { + const char* const pair_end = std::find(pair_begin, end, ';'); + const auto pair = range(pair_begin, pair_end); + if (pair.empty()) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + + const auto kv_separator = std::find(pair_begin, pair_end, ':'); + if (kv_separator == pair_end) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + + const auto key = range(pair_begin, kv_separator); + const auto value = range(kv_separator + 1, pair_end); + if (key == "o") { + result.origin = std::string{value}; + } else if (key == "s") { + const auto maybe_priority = parse_int(value, 10); + if (!maybe_priority) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + const int priority = *maybe_priority; + // If we didn't parse a sampling priority from traceparent, or if the one + // we just parsed from tracestate is consistent with the previous, then + // set the sampling priority to the one we just parsed. + // Alternatively, if we already parsed a sampling priority from + // traceparent and got a result inconsistent with that parsed here, go + // with the one previously parsed from traceparent. + if (!result.sampling_priority || + (*result.sampling_priority > 0) == (priority > 0)) { + result.sampling_priority = priority; + } + } else if (starts_with(key, "t.")) { + // The part of the key that follows "t." is the name of a trace tag, + // except without the "_dd.p." prefix. + const auto tag_suffix = key.substr(2); + std::string tag_name = "_dd.p."; + append(tag_name, tag_suffix); + result.trace_tags.insert_or_assign(std::move(tag_name), + std::string{value}); + } else { + // Unrecognized key: append the whole pair to + // `additional_datadog_w3c_tracestate`, which will be used if/when we + // inject trace context. + auto& entries = result.additional_datadog_w3c_tracestate; + if (!entries) { + entries.emplace(); + } + if (!entries->empty()) { + *entries += ';'; + } + append(*entries, pair); + } + + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + } + + return nullopt; +} + // TODO: document Optional extract_tracestate(ExtractedData& result, const DictReader& headers) { - // TODO - (void)result; - (void)headers; - return nullopt; + const auto maybe_tracestate = headers.lookup("tracestate"); + if (!maybe_tracestate) { + return nullopt; + } + + const auto tracestate = strip(*maybe_tracestate); + auto maybe_parsed = parse_tracestate(tracestate); + if (!maybe_parsed) { + // No "dd" entry in `tracestate`, so there's nothing to extract. + result.additional_w3c_tracestate = std::string{tracestate}; + return nullopt; + } + + auto& [datadog_value, other_entries] = *maybe_parsed; + result.additional_datadog_w3c_tracestate = std::move(other_entries); + return parse_datadog_tracestate(result, datadog_value); } } // namespace Expected extract_w3c( const DictReader& headers, - std::unordered_map& span_tags) { + std::unordered_map& span_tags, Logger&) { ExtractedData result; if (auto error_tag_value = extract_traceparent(result, headers)) { diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 26e594dc..3d89ad02 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -14,6 +14,7 @@ namespace datadog { namespace tracing { class DictReader; +class Logger; // Return `ExtractedData` deduced from the "traceparent" and "tracestate" // entries of the specified `headers`. If an error occurs, set a value for the @@ -22,7 +23,7 @@ class DictReader; // `ExtractedData` when extraction fails. Expected extract_w3c( const DictReader& headers, - std::unordered_map& span_tags); + std::unordered_map& span_tags, Logger&); // Return a value for the "traceparent" header consisting of the specified // `trace_id` or the optionally specified `full_w3c_trace_id_hex` as the trace From 3158907ec24c948cd1978ca7b65e95ee91c788f3 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 11:48:38 -0500 Subject: [PATCH 10/54] TODO: AMEND: tested tracestate extraction --- src/datadog/w3c_propagation.cpp | 86 ++++++-------- test/mocks/loggers.cpp | 16 +++ test/mocks/loggers.h | 3 + test/test.cpp | 8 ++ test/test.h | 2 + test/test_tracer.cpp | 203 ++++++++++++++++++++++++++++++++ 6 files changed, 266 insertions(+), 52 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 2e017e68..45a49633 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -1,5 +1,6 @@ #include "w3c_propagation.h" +#include #include #include #include @@ -27,7 +28,7 @@ Optional extract_traceparent(ExtractedData& result, const auto traceparent = strip(*maybe_traceparent); // Note that leading and trailing whitespace was already removed above. - // Note that the match group 0 is the entire match. + // Note that match group 0 is the entire match. static const auto& pattern = "([0-9a-f]{2})" // hex version number (match group 1) "-" @@ -89,16 +90,13 @@ Optional parse_tracestate(StringView tracestate) { const char* const begin = tracestate.begin(); const char* const end = tracestate.end(); const char* pair_begin = begin; - for (;;) { + while (pair_begin != end) { const char* const pair_end = std::find(pair_begin, end, ','); // Note that since this `pair` is `strip`ped, `pair_begin` is not // necessarily equal to `pair.begin()` (similarly for the ends). const auto pair = strip(range(pair_begin, pair_end)); if (pair.empty()) { - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -107,20 +105,14 @@ Optional parse_tracestate(StringView tracestate) { // This is an invalid entry because it contains a non-whitespace character // but not a "=". // Let's move on to the next entry. - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const auto key = range(pair.begin(), kv_separator); if (key != "dd") { // On to the next. - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -128,7 +120,7 @@ Optional parse_tracestate(StringView tracestate) { result.emplace(); result->datadog_value = range(kv_separator + 1, pair.end()); // `result->other_entries` is whatever was before the "dd" entry and - // whatever is after the "dd" entry, but without a redundant comma in the + // whatever is after the "dd" entry, but without an extra comma in the // middle. if (pair_begin != begin) { // There's a prefix @@ -142,35 +134,30 @@ Optional parse_tracestate(StringView tracestate) { append(result->other_entries, range(pair_end + 1, end)); } - return result; + break; } + + return result; } // TODO: document -Optional parse_datadog_tracestate(ExtractedData& result, - StringView datadog_value) { +void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { const char* const begin = datadog_value.begin(); const char* const end = datadog_value.end(); const char* pair_begin = begin; - for (;;) { + while (pair_begin != end) { const char* const pair_end = std::find(pair_begin, end, ';'); const auto pair = range(pair_begin, pair_end); if (pair.empty()) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const auto kv_separator = std::find(pair_begin, pair_end, ':'); if (kv_separator == pair_end) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -182,10 +169,7 @@ Optional parse_datadog_tracestate(ExtractedData& result, const auto maybe_priority = parse_int(value, 10); if (!maybe_priority) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const int priority = *maybe_priority; @@ -205,8 +189,12 @@ Optional parse_datadog_tracestate(ExtractedData& result, const auto tag_suffix = key.substr(2); std::string tag_name = "_dd.p."; append(tag_name, tag_suffix); + // The tag value was encoded with all '=' replaced by '~'. Undo that + // transformation. + std::string decoded_value{value}; + std::replace(decoded_value.begin(), decoded_value.end(), '~', '='); result.trace_tags.insert_or_assign(std::move(tag_name), - std::string{value}); + std::move(decoded_value)); } else { // Unrecognized key: append the whole pair to // `additional_datadog_w3c_tracestate`, which will be used if/when we @@ -214,41 +202,39 @@ Optional parse_datadog_tracestate(ExtractedData& result, auto& entries = result.additional_datadog_w3c_tracestate; if (!entries) { entries.emplace(); - } - if (!entries->empty()) { + } else { *entries += ';'; } append(*entries, pair); } - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; } - - return nullopt; } // TODO: document -Optional extract_tracestate(ExtractedData& result, - const DictReader& headers) { +void extract_tracestate(ExtractedData& result, const DictReader& headers) { const auto maybe_tracestate = headers.lookup("tracestate"); if (!maybe_tracestate) { - return nullopt; + return; } const auto tracestate = strip(*maybe_tracestate); auto maybe_parsed = parse_tracestate(tracestate); if (!maybe_parsed) { // No "dd" entry in `tracestate`, so there's nothing to extract. - result.additional_w3c_tracestate = std::string{tracestate}; - return nullopt; + if (!tracestate.empty()) { + result.additional_w3c_tracestate = std::string{tracestate}; + } + return; } auto& [datadog_value, other_entries] = *maybe_parsed; - result.additional_datadog_w3c_tracestate = std::move(other_entries); - return parse_datadog_tracestate(result, datadog_value); + if (!other_entries.empty()) { + result.additional_w3c_tracestate = std::move(other_entries); + } + + parse_datadog_tracestate(result, datadog_value); } } // namespace @@ -270,11 +256,7 @@ Expected extract_w3c( return result; } - if (auto error_tag_value = extract_tracestate(result, headers)) { - span_tags[tags::internal::w3c_extraction_error] = - std::move(*error_tag_value); - // Carry on with whatever data was extracted from traceparent. - } + extract_tracestate(result, headers); return result; } diff --git a/test/mocks/loggers.cpp b/test/mocks/loggers.cpp index a7655dfd..795a8a75 100644 --- a/test/mocks/loggers.cpp +++ b/test/mocks/loggers.cpp @@ -1 +1,17 @@ #include "loggers.h" + +#include +#include + +std::ostream& operator<<(std::ostream& stream, + const std::vector& entries) { + stream << ""; + for (std::size_t i = 0; i < entries.size(); ++i) { + const auto& entry = entries[i]; + const auto kind_name = + entry.kind == MockLogger::Entry::ERROR ? "ERROR" : "STARTUP"; + stream << '\n' << (i + 1) << ". " << kind_name << ": "; + std::visit([&](const auto& value) { stream << value; }, entry.payload); + } + return stream << ""; +} diff --git a/test/mocks/loggers.h b/test/mocks/loggers.h index 5364a1c4..c41b4278 100644 --- a/test/mocks/loggers.h +++ b/test/mocks/loggers.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -97,3 +98,5 @@ struct MockLogger : public Logger { return std::get(found->payload); } }; + +std::ostream& operator<<(std::ostream&, const std::vector&); diff --git a/test/test.cpp b/test/test.cpp index ca57ed79..801d68a7 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -14,4 +14,12 @@ std::ostream& operator<<( return stream << item.value_or(""); } +std::ostream& operator<<(std::ostream& stream, + const std::optional& maybe) { + if (maybe) { + return stream << *maybe; + } + return stream << ""; +} + } // namespace std diff --git a/test/test.h b/test/test.h index e9a8664a..0cf17b1b 100644 --- a/test/test.h +++ b/test/test.h @@ -26,6 +26,8 @@ std::ostream& operator<<(std::ostream& stream, std::ostream& operator<<(std::ostream& stream, const std::optional& maybe); +std::ostream& operator<<(std::ostream& stream, const std::optional& maybe); + } // namespace std namespace datadog { diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 981c2b35..7142c17d 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -651,6 +652,208 @@ TEST_CASE("span extraction") { } } + SECTION("W3C tracestate extraction") { + // Ideally this would test the _behavior_ of W3C tracestate extraction, + // rather than its implementation. + // However, some of the effects of W3C tracestate extraction cannot be + // observed except by injecting trace context, and there's a separate test + // for W3C tracestate injection (in `test_span.cpp`). + // Here we test the tracestate portion of the `extract_w3c` function, + // declared in `w3c_propagation.h`. + struct TestCase { + int line; + std::string name; + std::string traceparent; + Optional tracestate; + Optional expected_sampling_priority = {}; + Optional expected_origin = {}; + std::unordered_map expected_trace_tags = {}; + Optional expected_additional_w3c_tracestate = {}; + Optional expected_additional_datadog_w3c_tracestate = {}; + }; + + static const std::string traceparent_prefix = + "00-00000000000000000000000000000001-0000000000000001-0"; + static const std::string traceparent_drop = traceparent_prefix + "0"; + static const std::string traceparent_keep = traceparent_prefix + "1"; + // clang-format off + auto test_case = GENERATE(values({ + {__LINE__, "no tracestate", + traceparent_drop, // traceparent + nullopt, // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "empty tracestate", + traceparent_drop, // traceparent + "", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "no dd entry", + traceparent_drop, // traceparent + "foo=hello,@thingy/thing=wah;wah;wah", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,@thingy/thing=wah;wah;wah", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "empty entry", + traceparent_drop, // traceparent + "foo=hello,,bar=thing", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,,bar=thing", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "malformed entry", + traceparent_drop, // traceparent + "foo=hello,chicken,bar=thing", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,chicken,bar=thing", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff before dd entry", + traceparent_drop, // traceparent + "foo=hello,bar=baz,dd=", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff after dd entry", + traceparent_drop, // traceparent + "dd=,foo=hello,bar=baz", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff before and after dd entry", + traceparent_drop, // traceparent + "chicken=yes,nuggets=yes,dd=,foo=hello,bar=baz", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "chicken=yes,nuggets=yes,foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "dd entry with empty subentries", + traceparent_drop, // traceparent + "dd=foo:bar;;;;;baz:bam;;;", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "foo:bar;baz:bam"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "dd entry with malformed subentries", + traceparent_drop, // traceparent + "dd=foo:bar;chicken;chicken;baz:bam;chicken", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "foo:bar;baz:bam"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "origin, trace tags, and extra fields", + traceparent_drop, // traceparent + "dd=o:France;t.foo:thing1;t.bar:thing2;x:wow;y:wow", // tracestate + 0, // expected_sampling_priority + "France", // expected_origin + {{"_dd.p.foo", "thing1"}, {"_dd.p.bar", "thing2"}}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "x:wow;y:wow"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "traceparent and tracestate sampling agree (1/4)", + traceparent_drop, // traceparent + "dd=s:0", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (2/4)", + traceparent_drop, // traceparent + "dd=s:-1", // tracestate + -1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (3/4)", + traceparent_keep, // traceparent + "dd=s:1", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (4/4)", + traceparent_keep, // traceparent + "dd=s:2", // tracestate + 2}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (1/4)", + traceparent_drop, // traceparent + "dd=s:1", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (2/4)", + traceparent_drop, // traceparent + "dd=s:2", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (3/4)", + traceparent_keep, // traceparent + "dd=s:0", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (4/4)", + traceparent_keep, // traceparent + "dd=s:-1", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "invalid sampling priority (1/2)", + traceparent_drop, // traceparent + "dd=s:oops", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "invalid sampling priority (2/2)", + traceparent_keep, // traceparent + "dd=s:oops", // tracestate + 1}, // expected_sampling_priority + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + CAPTURE(test_case.traceparent); + CAPTURE(test_case.tracestate); + + std::unordered_map span_tags; + MockLogger logger; + CAPTURE(logger.entries); + CAPTURE(span_tags); + + std::unordered_map headers; + headers["traceparent"] = test_case.traceparent; + if (test_case.tracestate) { + headers["tracestate"] = *test_case.tracestate; + } + MockDictReader reader{headers}; + + const auto extracted = extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + + REQUIRE(extracted->origin == test_case.expected_origin); + REQUIRE(extracted->trace_tags == test_case.expected_trace_tags); + REQUIRE(extracted->sampling_priority == + test_case.expected_sampling_priority); + REQUIRE(extracted->additional_w3c_tracestate == + test_case.expected_additional_w3c_tracestate); + REQUIRE(extracted->additional_datadog_w3c_tracestate == + test_case.expected_additional_datadog_w3c_tracestate); + + REQUIRE(logger.entries.empty()); + REQUIRE(span_tags.empty()); + } + SECTION("x-datadog-tags") { auto finalized_config = finalize_config(config); REQUIRE(finalized_config); From b69ad49b82bcb630a8d0075d3977d1624929cc2d Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 15:29:08 -0500 Subject: [PATCH 11/54] TODO: AMEND: add some more documentation --- src/datadog/extracted_data.h | 22 ++++++++++++++++++---- src/datadog/w3c_propagation.cpp | 24 ++++++++++++++++++++---- src/datadog/w3c_propagation.h | 4 +++- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index bfdd2dea..f6d90f82 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -1,6 +1,7 @@ #pragma once -// TODO: document +// This component provides a `struct`, `ExtractedData`, that stores fields +// extracted from trace context. It's an implementation detail of this library. #include #include @@ -17,11 +18,24 @@ struct ExtractedData { Optional origin; std::unordered_map trace_tags; Optional sampling_priority; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C` + // and trace context was successfully extracted, then `full_w3c_trace_id_hex` + // contains the hex-encoded 128-bit trace ID. `trace_id` will be the least + // significant 64 bits of the same value. `full_w3c_trace_id_hex` is used for + // the `W3C` injection style. Optional full_w3c_trace_id_hex; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, + // then `additional_w3c_tracestate` contains the parts of the "tracestate" + // header that are not the "dd" (Datadog) entry. If there are no other parts, + // then `additional_w3c_tracestate` is null. + // `additional_w3c_tracestate` is used for the `W3C` injection style. Optional additional_w3c_tracestate; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, + // and if the "tracestate" header contained a "dd" (Datadog) entry, then + // `additional_datadog_w3c_tracestate` contains fields from within the "dd" + // entry that were not interpreted. If there are no such fields, then + // `additional_datadog_w3c_tracestate` is null. + // `additional_datadog_w3c_tracestate` is used for the `W3C` injection style. Optional additional_datadog_w3c_tracestate; }; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 45a49633..ec69d4c5 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -77,13 +77,16 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } -// TODO: document +// `struct PartiallyParsedTracestat` contains the separated Datadog-specific and +// non-Datadog-specific portions of tracestate. struct PartiallyParsedTracestate { StringView datadog_value; std::string other_entries; }; -// TODO: document +// Return the separate Datadog-specific and non-Datadog-specific portions of the +// specified `tracestate`. If `tracestate` does not have a Datadog-specific +// portion, return `nullopt`. Optional parse_tracestate(StringView tracestate) { Optional result; @@ -140,7 +143,15 @@ Optional parse_tracestate(StringView tracestate) { return result; } -// TODO: document +// Fill the specified `result` with information parsed from the specified +// `datadog_value`. `datadog_value` is the value of the "dd" entry in the +// "tracestate" header. +// +// `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// - `origin` +// - `trace_tags` +// - `sampling_priority` +// - `additional_datadog_w3c_tracestate` void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { const char* const begin = datadog_value.begin(); const char* const end = datadog_value.end(); @@ -212,7 +223,12 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { } } -// TODO: document +// Fill the specified `result` with information parsed from the "tracestate" +// element of the specified `headers`, if present. +// +// `extract_tracestate` populates the `additional_w3c_tracestate` field of +// `ExtractedData`, in addition to those populated by +// `parse_datadog_tracestate`. void extract_tracestate(ExtractedData& result, const DictReader& headers) { const auto maybe_tracestate = headers.lookup("tracestate"); if (!maybe_tracestate) { diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 3d89ad02..16096a5b 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -1,6 +1,8 @@ #pragma once -// TODO: document +// This component provides functions for extracting and injecting trace context +// in the `PropagationStyle::W3C` style. These functions decode and encode the +// "traceparent" and "tracestate" HTTP request headers. #include #include From fea6cae4f017f72d4198f63acaa91b9605bdb873 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 16:12:59 -0500 Subject: [PATCH 12/54] TODO: AMEND: checkpoint --- src/datadog/trace_segment.cpp | 89 ++++++++++++++++++++++----------- src/datadog/w3c_propagation.cpp | 37 ++++++++++++++ src/datadog/w3c_propagation.h | 7 +++ 3 files changed, 103 insertions(+), 30 deletions(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 214960f0..0b67ab4d 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -21,6 +21,38 @@ namespace datadog { namespace tracing { +namespace { + +// Encode the specified `trace_tags`. If the encoded value is not longer than +// the specified `tags_header_max_size`, then set it as the "x-datadog-tags" +// header using the specified `writer`. If the encoded value is oversized, then +// write a diagnostic to the specified `logger` and set a propagation error tag +// on the specified `local_root_tags`. +void inject_trace_tags( + DictWriter& writer, + const std::unordered_map& trace_tags, + std::size_t tags_header_max_size, + std::unordered_map& local_root_tags, + Logger& logger) { + const std::string encoded_trace_tags = encode_tags(trace_tags); + + if (encoded_trace_tags.size() > tags_header_max_size) { + std::string message; + message += + "Serialized x-datadog-tags header value is too large. The configured " + "maximum size is "; + message += std::to_string(tags_header_max_size); + message += " bytes, but the encoded value is "; + message += std::to_string(encoded_trace_tags.size()); + message += " bytes."; + logger.log_error(message); + local_root_tags[tags::internal::propagation_error] = "inject_max_size"; + } else if (!encoded_trace_tags.empty()) { + writer.set("x-datadog-tags", encoded_trace_tags); + } +} + +} // namespace TraceSegment::TraceSegment( const std::shared_ptr& logger, @@ -201,41 +233,24 @@ void TraceSegment::update_decision_maker_trace_tag() { } void TraceSegment::inject(DictWriter& writer, const SpanData& span) { + // If the only injection style is `NONE`, then don't do anything. + if (injection_styles_.size() == 1 && + injection_styles_[0] == PropagationStyle::NONE) { + return; + } + + // The sampling priority can change (it can be overridden on another thread), + // and trace tags might change when that happens ("_dd.p.dm"). + // So, we lock here, make a sampling decision if necessary, and then copy the + // decision and trace tags before unlocking. int sampling_priority; - std::string encoded_trace_tags; + std::unordered_map trace_tags; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); assert(sampling_decision_); sampling_priority = sampling_decision_->priority; - encoded_trace_tags = encode_tags(trace_tags_); - } - - // Origin and trace tag headers are always propagated, unless the only - // injection style is "none". - // Other headers depend on the injection styles. - if (injection_styles_.size() == 1 && - injection_styles_[0] == PropagationStyle::NONE) { - return; - } - - if (origin_) { - writer.set("x-datadog-origin", *origin_); - } - if (encoded_trace_tags.size() > tags_header_max_size_) { - std::string message; - message += - "Serialized x-datadog-tags header value is too large. The configured " - "maximum size is "; - message += std::to_string(tags_header_max_size_); - message += " bytes, but the encoded value is "; - message += std::to_string(encoded_trace_tags.size()); - message += " bytes."; - logger_->log_error(message); - SpanData& local_root = *spans_.front(); - local_root.tags[tags::internal::propagation_error] = "inject_max_size"; - } else if (!encoded_trace_tags.empty()) { - writer.set("x-datadog-tags", encoded_trace_tags); + trace_tags = trace_tags_; } for (const auto style : injection_styles_) { @@ -245,17 +260,31 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", std::to_string(sampling_priority)); + if (origin_) { + writer.set("x-datadog-origin", *origin_); + } + inject_trace_tags(writer, trace_tags, tags_header_max_size_, + spans_.front()->tags, *logger_); break; case PropagationStyle::B3: writer.set("x-b3-traceid", hex(span.trace_id)); writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); + if (origin_) { + writer.set("x-datadog-origin", *origin_); + } + inject_trace_tags(writer, trace_tags, tags_header_max_size_, + spans_.front()->tags, *logger_); break; case PropagationStyle::W3C: writer.set("traceparent", encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, span.span_id, sampling_priority)); - // TODO writer.set("tracestate", ...); + // TODO: handle oversized things. + writer.set("tracestate", + encode_tracestate(sampling_priority, origin_, trace_tags, + additional_datadog_w3c_tracestate_, + additional_w3c_tracestate_)); break; default: assert(style == PropagationStyle::NONE); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index ec69d4c5..c7b54f57 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -306,5 +306,42 @@ std::string encode_traceparent( return result; } +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate) { + std::string result = "dd=s:"; + result += std::to_string(sampling_priority); + + if (origin) { + result += ";o:"; + result += *origin; + } + + for (const auto& [key, value] : trace_tags) { + // `key` is "_dd.p.", but we want "t.". + result += ";t."; + result.append(key, sizeof("_dd.p.") - 1); + + // `value` might contain equal signs ("="), which is reserved in tracestate. + // Replace them with tildes ("~"). + result += value; + std::replace(result.end() - value.size(), result.end(), '=', '~'); + } + + if (additional_datadog_w3c_tracestate) { + result += ';'; + result += *additional_datadog_w3c_tracestate; + } + + if (additional_w3c_tracestate) { + result += ','; + result += *additional_w3c_tracestate; + } + + return result; +} + } // namespace tracing } // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 16096a5b..fe11724e 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -35,5 +35,12 @@ std::string encode_traceparent( std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, std::uint64_t span_id, int sampling_priority); +// Return a value for the "tracestate" header containing the specified fields. +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate); + } // namespace tracing } // namespace datadog From b1888ec165af07b92924a6b4d6d62fb7e642cd20 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 17:12:37 -0500 Subject: [PATCH 13/54] TODO: AMEND: checkpoint --- src/datadog/trace_segment.cpp | 1 - src/datadog/w3c_propagation.cpp | 63 ++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 0b67ab4d..8b2ec50e 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -280,7 +280,6 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("traceparent", encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, span.span_id, sampling_priority)); - // TODO: handle oversized things. writer.set("tracestate", encode_tracestate(sampling_priority, origin_, trace_tags, additional_datadog_w3c_tracestate_, diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index c7b54f57..526f258a 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -15,6 +15,22 @@ namespace datadog { namespace tracing { namespace { +// Return a predicate that returns whether its `char` argument is any of the +// following: +// +// - outside of the ASCII inclusive range `[lowest_ascii, highest_ascii]` +// - equal to one of the `disallowed_characters`. +// +// `verboten` is used as an argument to `std::replace_if` to sanitize field +// values within the tracestate header. +auto verboten(int lowest_ascii, int highest_ascii, + StringView disallowed_characters) { + return [=, chars = disallowed_characters](char ch) { + return int(ch) < lowest_ascii || int(ch) > highest_ascii || + std::find(chars.begin(), chars.end(), ch) != chars.end(); + }; +} + // Populate the specified `result` with data extracted from the "traceparent" // entry of the specified `headers`. Return `nullopt` on success. Return a value // for the `tags::internal::w3c_extraction_error` tag if an error occurs. @@ -148,6 +164,7 @@ Optional parse_tracestate(StringView tracestate) { // "tracestate" header. // // `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// // - `origin` // - `trace_tags` // - `sampling_priority` @@ -306,28 +323,49 @@ std::string encode_traceparent( return result; } -std::string encode_tracestate( +std::string encode_datadog_tracestate( int sampling_priority, const Optional& origin, const std::unordered_map& trace_tags, - const Optional& additional_datadog_w3c_tracestate, - const Optional& additional_w3c_tracestate) { + const Optional& additional_datadog_w3c_tracestate) { std::string result = "dd=s:"; result += std::to_string(sampling_priority); + const std::size_t max_size = 256; + std::size_t last_good_size = result.size(); + if (origin) { result += ";o:"; result += *origin; + std::replace_if(result.end() - origin->size(), result.end(), + verboten(0x20, 0x7e, ",;="), '_'); } + if (result.size() > max_size) { + result.resize(last_good_size); + return result; + } + last_good_size = result.size(); + for (const auto& [key, value] : trace_tags) { // `key` is "_dd.p.", but we want "t.". result += ";t."; - result.append(key, sizeof("_dd.p.") - 1); + const auto prefix_length = sizeof("_dd.p.") - 1; + result.append(key, prefix_length); + std::replace_if(result.end() - (key.size() - prefix_length), result.end(), + verboten(0x20, 0x7e, " ,;="), '_'); + result += value; + std::replace_if(result.end() - value.size(), result.end(), + verboten(0x20, 0x7e, ",;~"), '_'); // `value` might contain equal signs ("="), which is reserved in tracestate. // Replace them with tildes ("~"). - result += value; std::replace(result.end() - value.size(), result.end(), '=', '~'); + + if (result.size() > max_size) { + result.resize(last_good_size); + return result; + } + last_good_size = result.size(); } if (additional_datadog_w3c_tracestate) { @@ -335,6 +373,21 @@ std::string encode_tracestate( result += *additional_datadog_w3c_tracestate; } + if (result.size() > max_size) { + result.resize(last_good_size); + } + + return result; +} + +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate) { + std::string result = encode_datadog_tracestate( + sampling_priority, origin, trace_tags, additional_datadog_w3c_tracestate); + if (additional_w3c_tracestate) { result += ','; result += *additional_w3c_tracestate; From 6bc31683e867b19437f94288f1d2644d8ea6392f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 19:46:00 -0500 Subject: [PATCH 14/54] TODO: AMEND: checkpoint, before I mess with flat maps --- src/datadog/tags.cpp | 1 - src/datadog/tags.h | 1 - src/datadog/w3c_propagation.cpp | 17 ++++--- test/mocks/loggers.cpp | 2 +- test/test_span.cpp | 85 +++++++++++++++++++++++++++++++++ test/test_tracer_config.cpp | 18 +++---- 6 files changed, 106 insertions(+), 18 deletions(-) diff --git a/src/datadog/tags.cpp b/src/datadog/tags.cpp index 84b91202..db5828b2 100644 --- a/src/datadog/tags.cpp +++ b/src/datadog/tags.cpp @@ -29,7 +29,6 @@ const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism"; const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate"; const std::string span_sampling_limit = "_dd.span_sampling.max_per_second"; const std::string w3c_extraction_error = "_dd.w3c_extraction_error"; -const std::string w3c_injection_error = "_dd.w3c_injection_error"; } // namespace internal diff --git a/src/datadog/tags.h b/src/datadog/tags.h index b1d77c58..996cc58b 100644 --- a/src/datadog/tags.h +++ b/src/datadog/tags.h @@ -33,7 +33,6 @@ extern const std::string span_sampling_mechanism; extern const std::string span_sampling_rule_rate; extern const std::string span_sampling_limit; extern const std::string w3c_extraction_error; -extern const std::string w3c_injection_error; } // namespace internal // Return whether the specified `tag_name` is reserved for use internal to this diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 526f258a..87935829 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -21,8 +21,8 @@ namespace { // - outside of the ASCII inclusive range `[lowest_ascii, highest_ascii]` // - equal to one of the `disallowed_characters`. // -// `verboten` is used as an argument to `std::replace_if` to sanitize field -// values within the tracestate header. +// `verboten` is used with `std::replace_if` to sanitize field values within the +// tracestate header. auto verboten(int lowest_ascii, int highest_ascii, StringView disallowed_characters) { return [=, chars = disallowed_characters](char ch) { @@ -347,13 +347,18 @@ std::string encode_datadog_tracestate( last_good_size = result.size(); for (const auto& [key, value] : trace_tags) { - // `key` is "_dd.p.", but we want "t.". + const StringView prefix = "_dd.p."; + if (!starts_with(key, prefix)) { + continue; + } + + // `key` is "_dd.p.", but we want "t.". result += ";t."; - const auto prefix_length = sizeof("_dd.p.") - 1; - result.append(key, prefix_length); - std::replace_if(result.end() - (key.size() - prefix_length), result.end(), + result.append(key, prefix.size()); + std::replace_if(result.end() - (key.size() - prefix.size()), result.end(), verboten(0x20, 0x7e, " ,;="), '_'); + result += ':'; result += value; std::replace_if(result.end() - value.size(), result.end(), verboten(0x20, 0x7e, ",;~"), '_'); diff --git a/test/mocks/loggers.cpp b/test/mocks/loggers.cpp index 795a8a75..b7c7e962 100644 --- a/test/mocks/loggers.cpp +++ b/test/mocks/loggers.cpp @@ -13,5 +13,5 @@ std::ostream& operator<<(std::ostream& stream, stream << '\n' << (i + 1) << ". " << kind_name << ": "; std::visit([&](const auto& value) { stream << value; }, entry.payload); } - return stream << ""; + return stream << "\n"; } diff --git a/test/test_span.cpp b/test/test_span.cpp index 2acc13b3..be331201 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -544,3 +544,88 @@ TEST_CASE("injecting W3C traceparent header") { REQUIRE(found->second == expected); } } + +TEST_CASE("injecting W3C tracestate header") { + // Concerns: + // - the basics: + // - sampling priority + // - origin + // - trace tags + // - extra fields (extracted from W3C) + // - all of the above + // - character substitutions: + // - in origin + // - in trace tag key + // - in trace tag value + // - special tilde ("~") behavior + // - length limit: + // - at origin + // - at a trace tag (extract extra fields from W3C) + // - at the extra fields (extracted from W3C) + + TracerConfig config; + config.defaults.service = "testsvc"; + // The order of the extraction styles doesn't matter for this test, because + // it'll either be one or the other in the test cases. + config.extraction_styles = {PropagationStyle::DATADOG, PropagationStyle::W3C}; + config.injection_styles = {PropagationStyle::W3C}; + // If one of these test cases results in a local sampling decision, let it be + // "drop." + config.trace_sampler.sample_rate = 0.0; + const auto logger = std::make_shared(); + config.logger = logger; + config.collector = std::make_shared(); + + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + Tracer tracer{*finalized_config}; + + struct TestCase { + int line; + std::string name; + std::unordered_map input_headers; + std::string expected_tracestate; + }; + + // clang-format off + auto test_case = GENERATE(values({ + {__LINE__, "sampling priority", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-sampling-priority", "2"}}, + "dd=s:2"}, + + {__LINE__, "origin", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "France"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;o:France"}, + + {__LINE__, "trace tags", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.foo:x;t.bar:y"}, + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + CAPTURE(test_case.input_headers); + CAPTURE(test_case.expected_tracestate); + CAPTURE(logger->entries); + + MockDictReader reader{test_case.input_headers}; + const auto span = tracer.extract_span(reader); + REQUIRE(span); + + MockDictWriter writer; + span->inject(writer); + + CAPTURE(writer.items); + const auto found = writer.items.find("tracestate"); + REQUIRE(found != writer.items.end()); + + REQUIRE(found->second == test_case.expected_tracestate); + + REQUIRE(logger->error_count() == 0); +} diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 27e4bb85..a106875f 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -1158,19 +1158,19 @@ TEST_CASE("TracerConfig propagation styles") { const char* const vars[] = {ts, tse, se, tsi, si}; constexpr auto n = sizeof(vars) / sizeof(vars[0]); // clang-format off - const bool x = false; // ignored values - const bool expect_warning[n][n] = { - // ts tse se tsi si - // --- --- --- --- --- - /* ts */{ x, true, true, true, true }, + const bool x = false; // ignored values + const bool expect_warning[n][n] = { + // ts tse se tsi si + // --- --- --- --- --- + /* ts */{ x, true, true, true, true }, - /* tse */{ x, x, true, false, false }, + /* tse */{ x, x, true, false, false }, - /* se */{ x, x, x, false, false }, + /* se */{ x, x, x, false, false }, - /* tsi */{ x, x, x, x, true }, + /* tsi */{ x, x, x, x, true }, - /* si */{ x, x, x, x, x }, + /* si */{ x, x, x, x, x }, }; // clang-format on for (std::size_t i = 0; i < n; ++i) { From cf67c0f26446ff2a46003ddfdd7cec54fb4da7e0 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 21:25:27 -0500 Subject: [PATCH 15/54] TODO: AMEND: trace_tags is a vector --- src/datadog/extracted_data.h | 5 +++-- src/datadog/tag_propagation.cpp | 11 +++++------ src/datadog/tag_propagation.h | 7 ++++--- src/datadog/trace_segment.cpp | 25 +++++++++++++++++++------ src/datadog/trace_segment.h | 6 +++--- src/datadog/tracer.cpp | 13 +++++++------ src/datadog/w3c_propagation.cpp | 8 ++++---- src/datadog/w3c_propagation.h | 2 +- test/matchers.h | 16 +++++++++++++++- test/test_span.cpp | 2 -- test/test_tracer.cpp | 2 +- test/test_tracer_config.cpp | 1 + 12 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index f6d90f82..0f182d7f 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -5,7 +5,8 @@ #include #include -#include +#include +#include #include "optional.h" @@ -16,7 +17,7 @@ struct ExtractedData { Optional trace_id; Optional parent_id; Optional origin; - std::unordered_map trace_tags; + std::vector> trace_tags; Optional sampling_priority; // If this `ExtractedData` was created on account of `PropagationStyle::W3C` // and trace context was successfully extracted, then `full_w3c_trace_id_hex` diff --git a/src/datadog/tag_propagation.cpp b/src/datadog/tag_propagation.cpp index 6c17be39..e3f024ec 100644 --- a/src/datadog/tag_propagation.cpp +++ b/src/datadog/tag_propagation.cpp @@ -32,7 +32,7 @@ namespace { // Insert into the specified `destination` a tag decoded from the specified // `entry`. Return an `Error` if an error occurs. Expected decode_tag( - std::unordered_map& destination, + std::vector>& destination, StringView entry) { const auto separator = std::find(entry.begin(), entry.end(), '='); if (separator == entry.end()) { @@ -44,8 +44,7 @@ Expected decode_tag( const StringView key = range(entry.begin(), separator); const StringView value = range(separator + 1, entry.end()); - // Among duplicate keys, most recent value wins. - destination.insert_or_assign(std::string(key), std::string(value)); + destination.emplace_back(std::string(key), std::string(value)); return nullopt; } @@ -59,9 +58,9 @@ void append_tag(std::string& serialized_tags, StringView tag_key, } // namespace -Expected> decode_tags( +Expected>> decode_tags( StringView header_value) { - std::unordered_map tags; + std::vector> tags; auto iter = header_value.begin(); const auto end = header_value.end(); @@ -88,7 +87,7 @@ Expected> decode_tags( } std::string encode_tags( - const std::unordered_map& trace_tags) { + const std::vector>& trace_tags) { std::string result; auto iter = trace_tags.begin(); if (iter == trace_tags.end()) { diff --git a/src/datadog/tag_propagation.h b/src/datadog/tag_propagation.h index b518716d..99666856 100644 --- a/src/datadog/tag_propagation.h +++ b/src/datadog/tag_propagation.h @@ -12,7 +12,8 @@ // "x-datadog-tags" header format. #include -#include +#include +#include #include "expected.h" #include "string_view.h" @@ -22,13 +23,13 @@ namespace tracing { // Return a name->value mapping of tags parsed from the specified // `header_value`, or return an `Error` if an error occurs. -Expected> decode_tags( +Expected>> decode_tags( StringView header_value); // Serialize the specified `trace_tags` into the propagation format and return // the resulting string. std::string encode_tags( - const std::unordered_map& trace_tags); + const std::vector>& trace_tags); } // namespace tracing } // namespace datadog diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 8b2ec50e..567259da 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "collector.h" #include "collector_response.h" @@ -30,7 +31,7 @@ namespace { // on the specified `local_root_tags`. void inject_trace_tags( DictWriter& writer, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, std::size_t tags_header_max_size, std::unordered_map& local_root_tags, Logger& logger) { @@ -63,7 +64,7 @@ TraceSegment::TraceSegment( const std::vector& injection_styles, const Optional& hostname, Optional origin, std::size_t tags_header_max_size, - std::unordered_map trace_tags, + std::vector> trace_tags, Optional sampling_decision, Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, @@ -224,11 +225,23 @@ void TraceSegment::update_decision_maker_trace_tag() { assert(sampling_decision_); + const auto found = std::find_if( + trace_tags_.begin(), trace_tags_.end(), [](const auto& entry) { + return entry.first == tags::internal::decision_maker; + }); + if (sampling_decision_->priority <= 0) { - trace_tags_.erase(tags::internal::decision_maker); + if (found != trace_tags_.end()) { + trace_tags_.erase(found); + } + return; + } + + auto value = "-" + std::to_string(*sampling_decision_->mechanism); + if (found == trace_tags_.end()) { + trace_tags_.emplace_back(tags::internal::decision_maker, std::move(value)); } else { - trace_tags_[tags::internal::decision_maker] = - "-" + std::to_string(*sampling_decision_->mechanism); + found->second == std::move(value); } } @@ -244,7 +257,7 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { // So, we lock here, make a sampling decision if necessary, and then copy the // decision and trace tags before unlocking. int sampling_priority; - std::unordered_map trace_tags; + std::vector> trace_tags; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); diff --git a/src/datadog/trace_segment.h b/src/datadog/trace_segment.h index 772e9c36..0a8ccc42 100644 --- a/src/datadog/trace_segment.h +++ b/src/datadog/trace_segment.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "expected.h" @@ -62,7 +62,7 @@ class TraceSegment { const Optional hostname_; const Optional origin_; const std::size_t tags_header_max_size_; - std::unordered_map trace_tags_; + std::vector> trace_tags_; std::vector> spans_; std::size_t num_finished_spans_; @@ -81,7 +81,7 @@ class TraceSegment { const std::vector& injection_styles, const Optional& hostname, Optional origin, std::size_t tags_header_max_size, - std::unordered_map trace_tags, + std::vector> trace_tags, Optional sampling_decision, Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 79d27dd3..dcc5c142 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -37,11 +37,12 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, if (auto* error = maybe_trace_tags.if_error()) { logger.log_error(*error); span_tags[tags::internal::propagation_error] = "decoding_error"; - } else { - for (const auto& [key, value] : *maybe_trace_tags) { - if (starts_with(key, "_dd.p.")) { - result.trace_tags.insert_or_assign(key, value); - } + return; + } + + for (auto& [key, value] : *maybe_trace_tags) { + if (starts_with(key, "_dd.p.")) { + result.trace_tags.emplace_back(std::move(key), std::move(value)); } } } @@ -248,7 +249,7 @@ Span Tracer::create_span(const SpanConfig& config) { const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, - std::unordered_map{} /* trace_tags */, + std::vector>{} /* trace_tags */, nullopt /* sampling_decision */, nullopt /* full_w3c_trace_id_hex */, nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 87935829..347c2140 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -221,8 +221,8 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { // transformation. std::string decoded_value{value}; std::replace(decoded_value.begin(), decoded_value.end(), '~', '='); - result.trace_tags.insert_or_assign(std::move(tag_name), - std::move(decoded_value)); + result.trace_tags.emplace_back(std::move(tag_name), + std::move(decoded_value)); } else { // Unrecognized key: append the whole pair to // `additional_datadog_w3c_tracestate`, which will be used if/when we @@ -325,7 +325,7 @@ std::string encode_traceparent( std::string encode_datadog_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate) { std::string result = "dd=s:"; result += std::to_string(sampling_priority); @@ -387,7 +387,7 @@ std::string encode_datadog_tracestate( std::string encode_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, const Optional& additional_w3c_tracestate) { std::string result = encode_datadog_tracestate( diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index fe11724e..57946ca9 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -38,7 +38,7 @@ std::string encode_traceparent( // Return a value for the "tracestate" header containing the specified fields. std::string encode_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, const Optional& additional_w3c_tracestate); diff --git a/test/matchers.h b/test/matchers.h index 88102906..83b14d88 100644 --- a/test/matchers.h +++ b/test/matchers.h @@ -2,6 +2,7 @@ #include #include +#include #include "test.h" @@ -9,13 +10,26 @@ template class ContainsSubset : public Catch::MatcherBase { const Map* subset_; + // `find` for when we're comparing with a `vector`. + template + static auto find(const std::vector& other, const LookupKey& key) { + return std::find_if(other.begin(), other.end(), + [&](const auto& entry) { return entry.first == key; }); + } + + // `find` for when we're comparing with an associative container. + template + static auto find(const Other& other, const LookupKey& key) { + return other.find(key); + } + public: ContainsSubset(const Map& subset) : subset_(&subset) {} bool match(const Map& other) const override { return std::all_of(subset_->begin(), subset_->end(), [&](const auto& item) { const auto& [key, value] = item; - auto found = other.find(key); + auto found = find(other, key); return found != other.end() && found->second == value; }); } diff --git a/test/test_span.cpp b/test/test_span.cpp index be331201..0582bf20 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -434,8 +434,6 @@ TEST_CASE("injection") { span->inject(writer); REQUIRE(writer.items.at("x-datadog-origin") == "Egypt"); - // Trace tags could get reordered (because we parse them into a hash - // table). So, compare the parsed versions. REQUIRE(writer.items.count("x-datadog-tags") == 1); const auto output = decode_tags(writer.items.at("x-datadog-tags")); const auto input = decode_tags(trace_tags); diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 7142c17d..0ad69d04 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -667,7 +667,7 @@ TEST_CASE("span extraction") { Optional tracestate; Optional expected_sampling_priority = {}; Optional expected_origin = {}; - std::unordered_map expected_trace_tags = {}; + std::vector> expected_trace_tags = {}; Optional expected_additional_w3c_tracestate = {}; Optional expected_additional_datadog_w3c_tracestate = {}; }; diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index a106875f..ab583228 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "mocks/collectors.h" #include "mocks/event_schedulers.h" From 0cff419166539811684cc492da186c4ade982eb8 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:00:08 -0500 Subject: [PATCH 16/54] TODO: AMEND: tracestate injection tested --- src/datadog/w3c_propagation.cpp | 25 ++++-------- test/test_span.cpp | 67 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 347c2140..a376dd26 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -330,9 +330,6 @@ std::string encode_datadog_tracestate( std::string result = "dd=s:"; result += std::to_string(sampling_priority); - const std::size_t max_size = 256; - std::size_t last_good_size = result.size(); - if (origin) { result += ";o:"; result += *origin; @@ -340,12 +337,6 @@ std::string encode_datadog_tracestate( verboten(0x20, 0x7e, ",;="), '_'); } - if (result.size() > max_size) { - result.resize(last_good_size); - return result; - } - last_good_size = result.size(); - for (const auto& [key, value] : trace_tags) { const StringView prefix = "_dd.p."; if (!starts_with(key, prefix)) { @@ -365,12 +356,6 @@ std::string encode_datadog_tracestate( // `value` might contain equal signs ("="), which is reserved in tracestate. // Replace them with tildes ("~"). std::replace(result.end() - value.size(), result.end(), '=', '~'); - - if (result.size() > max_size) { - result.resize(last_good_size); - return result; - } - last_good_size = result.size(); } if (additional_datadog_w3c_tracestate) { @@ -378,8 +363,14 @@ std::string encode_datadog_tracestate( result += *additional_datadog_w3c_tracestate; } - if (result.size() > max_size) { - result.resize(last_good_size); + const std::size_t max_size = 256; + while (result.size() > max_size) { + const auto last_semicolon_index = result.rfind(';'); + // This assumption is safe, because `result` always begins with + // "dd=s:", and that's fewer than `max_size` characters for any + // ``. + assert(last_semicolon_index != std::string::npos); + result.resize(last_semicolon_index); } return result; diff --git a/test/test_span.cpp b/test/test_span.cpp index 0582bf20..ada7df6f 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -558,7 +558,7 @@ TEST_CASE("injecting W3C tracestate header") { // - special tilde ("~") behavior // - length limit: // - at origin - // - at a trace tag (extract extra fields from W3C) + // - at a trace tag // - at the extra fields (extracted from W3C) TracerConfig config; @@ -585,6 +585,9 @@ TEST_CASE("injecting W3C tracestate header") { std::string expected_tracestate; }; + static const auto traceparent_drop = + "00-00000000000000000000000000000001-0000000000000001-00"; + // clang-format off auto test_case = GENERATE(values({ {__LINE__, "sampling priority", @@ -600,9 +603,69 @@ TEST_CASE("injecting W3C tracestate header") { {__LINE__, "trace tags", {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, - {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y"}}, + {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y,ignored=wrong_prefix"}}, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;t.foo:x;t.bar:y"}, + + {__LINE__, "extra fields", + {{"traceparent", traceparent_drop}, {"tracestate", "dd=foo:bar;boing:boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;foo:bar;boing:boing"}, + + {__LINE__, "all of the above", + {{"traceparent", traceparent_drop}, + {"tracestate", "dd=o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"}, + + {__LINE__, "replace invalid characters in origin", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "France, is a country=nation; so is 台北."}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;o:France_ is a country_nation_ so is ______."}, + + {__LINE__, "replace invalid characters in trace tag key", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.a;d台北x =foo,_dd.p.ok=bar"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.a_d______x_:foo;t.ok:bar"}, + + {__LINE__, "replace invalid characters in trace tag value", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.wacky=hello fr~d; how are คุณ?"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.wacky:hello fr_d_ how are _________?"}, + + {__LINE__, "replace equal signs with tildes in trace tag value", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.base64_thingy=d2Fra2EhIHdhaw=="}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.base64_thingy:d2Fra2EhIHdhaw~~"}, + + {__LINE__, "oversized origin truncates it and subsequent fields", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "long cat is looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"}, + {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.honk=honk"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1"}, + + {__LINE__, "oversized trace tag truncates it and subsequent fields", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.long_cat_is=looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong,_dd.p.lost=forever"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.foo:bar"}, + + {__LINE__, "oversized extra field truncates itself and subsequent fields", + {{"traceparent", traceparent_drop}, + {"tracestate", "dd=foo:bar;long_cat_is:looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong;lost:forever"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;foo:bar"}, + + {__LINE__, "non-Datadog tracestate", + {{"traceparent", traceparent_drop}, + {"tracestate", "foo=bar,boing=boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0,foo=bar,boing=boing"}, })); // clang-format on From 4e8719dc59bb07a358789ebd3f498208b9bd29bc Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:04:11 -0500 Subject: [PATCH 17/54] TODO: AMEND: fix scary typo --- src/datadog/trace_segment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 567259da..11040a66 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -241,7 +241,7 @@ void TraceSegment::update_decision_maker_trace_tag() { if (found == trace_tags_.end()) { trace_tags_.emplace_back(tags::internal::decision_maker, std::move(value)); } else { - found->second == std::move(value); + found->second = std::move(value); } } From a72b2977d443ce507e398c1a8e006b23c822776d Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:33:07 -0500 Subject: [PATCH 18/54] TODO: AMEND: update includes graph in doc/ --- doc/includes.dot | 673 +++++----- doc/includes.svg | 3362 ++++++++++++++++++++++++---------------------- 2 files changed, 2091 insertions(+), 1944 deletions(-) diff --git a/doc/includes.dot b/doc/includes.dot index 1725ee7f..a507fbdd 100644 --- a/doc/includes.dot +++ b/doc/includes.dot @@ -1,7 +1,7 @@ digraph G { - node0 [label="rate.cpp"]; - node1 [label="span_data.cpp"]; - node2 [label="threaded_event_scheduler.cpp"]; + node0 [label="threaded_event_scheduler.cpp"]; + node1 [label="rate.cpp"]; + node2 [label="span_data.cpp"]; node3 [label="default_http_client_null.cpp"]; node4 [label="trace_segment.h"]; node5 [label="tag_propagation.cpp"]; @@ -10,8 +10,8 @@ digraph G { node8 [label="string_view.h"]; node9 [label="span_sampler.h"]; node10 [label="dict_reader.cpp"]; - node11 [label="tags.h"]; - node12 [label="tracer_config.h"]; + node11 [label="tracer_config.h"]; + node12 [label="tags.h"]; node13 [label="curl/curl.h"]; node14 [label="expected.h"]; node15 [label="limiter.h"]; @@ -20,348 +20,369 @@ digraph G { node18 [label="sampling_mechanism.cpp"]; node19 [label="logger.cpp"]; node20 [label="rate.h"]; - node21 [label="sampling_util.h"]; - node22 [label="string_view.cpp"]; - node23 [label="sampling_decision.h"]; - node24 [label="span_data.h"]; - node25 [label="datadog_agent_config.h"]; - node26 [label="sampling_mechanism.h"]; - node27 [label="glob.h"]; - node28 [label="msgpack.cpp"]; - node29 [label="absl/types/optional.h"]; - node30 [label="default_http_client.h"]; - node31 [label="sampling_util.cpp"]; - node32 [label="datadog_agent_config.cpp"]; - node33 [label="span.cpp"]; - node34 [label="event_scheduler.cpp"]; - node35 [label="json_fwd.hpp"]; - node36 [label="datadog_agent.h"]; - node37 [label="propagation_style.cpp"]; - node38 [label="error.h"]; - node39 [label="event_scheduler.h"]; - node40 [label="tag_propagation.h"]; - node41 [label="clock.h"]; - node42 [label="logger.h"]; - node43 [label="span_defaults.h"]; - node44 [label="curl.h"]; - node45 [label="span_defaults.cpp"]; - node46 [label="threaded_event_scheduler.h"]; - node47 [label="tracer.h"]; - node48 [label="parse_util.h"]; - node49 [label="expected.cpp"]; - node50 [label="collector.cpp"]; - node51 [label="id_generator.cpp"]; - node52 [label="trace_sampler.cpp"]; - node53 [label="sampling_decision.cpp"]; - node54 [label="sampling_priority.cpp"]; - node55 [label="span_sampler.cpp"]; - node56 [label="collector.h"]; - node57 [label="curl.cpp"]; - node58 [label="null_collector.cpp"]; - node59 [label="cerr_logger.cpp"]; - node60 [label="collector_response.h"]; - node61 [label="msgpack.h"]; - node62 [label="trace_sampler.h"]; - node63 [label="span_config.cpp"]; - node64 [label="propagation_style.h"]; - node65 [label="dict_writer.cpp"]; - node66 [label="trace_sampler_config.cpp"]; - node67 [label="environment.h"]; - node68 [label="span_config.h"]; - node69 [label="datadog_agent.cpp"]; - node70 [label="winsock.h"]; - node71 [label="environment.cpp"]; - node72 [label="id_generator.h"]; - node73 [label="tracer_config.cpp"]; - node74 [label="net_util.cpp"]; - node75 [label="parse_util.cpp"]; - node76 [label="http_client.cpp"]; - node77 [label="optional.h"]; - node78 [label="null_collector.h"]; - node79 [label="charconv"]; - node80 [label="version.h"]; - node81 [label="span_sampler_config.h"]; - node82 [label="span_matcher.h"]; - node83 [label="version.cpp"]; - node84 [label="dict_reader.h"]; - node85 [label="span_matcher.cpp"]; - node86 [label="tracer.cpp"]; - node87 [label="default_http_client_curl.cpp"]; - node88 [label="http_client.h"]; - node89 [label="clock.cpp"]; - node90 [label="collector_response.cpp"]; - node91 [label="sampling_priority.h"]; - node92 [label="net_util.h"]; - node93 [label="limiter.cpp"]; - node94 [label="error.cpp"]; - node95 [label="absl/strings/string_view.h"]; - node96 [label="trace_sampler_config.h"]; - node97 [label="tags.cpp"]; - node98 [label="cerr_logger.h"]; - node99 [label="trace_segment.cpp"]; - node100 [label="span_sampler_config.cpp"]; + node21 [label="hex.cpp"]; + node22 [label="sampling_util.h"]; + node23 [label="string_view.cpp"]; + node24 [label="datadog_agent_config.h"]; + node25 [label="span_data.h"]; + node26 [label="sampling_decision.h"]; + node27 [label="sampling_mechanism.h"]; + node28 [label="glob.h"]; + node29 [label="msgpack.cpp"]; + node30 [label="absl/types/optional.h"]; + node31 [label="default_http_client.h"]; + node32 [label="sampling_util.cpp"]; + node33 [label="datadog_agent_config.cpp"]; + node34 [label="span.cpp"]; + node35 [label="event_scheduler.cpp"]; + node36 [label="json_fwd.hpp"]; + node37 [label="datadog_agent.h"]; + node38 [label="propagation_style.cpp"]; + node39 [label="error.h"]; + node40 [label="event_scheduler.h"]; + node41 [label="tag_propagation.h"]; + node42 [label="clock.h"]; + node43 [label="logger.h"]; + node44 [label="span_defaults.h"]; + node45 [label="curl.h"]; + node46 [label="span_defaults.cpp"]; + node47 [label="threaded_event_scheduler.h"]; + node48 [label="tracer.h"]; + node49 [label="parse_util.h"]; + node50 [label="expected.cpp"]; + node51 [label="collector.cpp"]; + node52 [label="extracted_data.cpp"]; + node53 [label="id_generator.cpp"]; + node54 [label="trace_sampler.cpp"]; + node55 [label="sampling_decision.cpp"]; + node56 [label="extracted_data.h"]; + node57 [label="sampling_priority.cpp"]; + node58 [label="collector.h"]; + node59 [label="span_sampler.cpp"]; + node60 [label="curl.cpp"]; + node61 [label="null_collector.cpp"]; + node62 [label="cerr_logger.cpp"]; + node63 [label="collector_response.h"]; + node64 [label="w3c_propagation.h"]; + node65 [label="msgpack.h"]; + node66 [label="trace_sampler.h"]; + node67 [label="span_config.cpp"]; + node68 [label="propagation_style.h"]; + node69 [label="dict_writer.cpp"]; + node70 [label="trace_sampler_config.cpp"]; + node71 [label="environment.h"]; + node72 [label="span_config.h"]; + node73 [label="datadog_agent.cpp"]; + node74 [label="w3c_propagation.cpp"]; + node75 [label="winsock.h"]; + node76 [label="environment.cpp"]; + node77 [label="id_generator.h"]; + node78 [label="tracer_config.cpp"]; + node79 [label="net_util.cpp"]; + node80 [label="parse_util.cpp"]; + node81 [label="http_client.cpp"]; + node82 [label="optional.h"]; + node83 [label="null_collector.h"]; + node84 [label="charconv"]; + node85 [label="version.h"]; + node86 [label="version.cpp"]; + node87 [label="span_matcher.h"]; + node88 [label="span_sampler_config.h"]; + node89 [label="dict_reader.h"]; + node90 [label="span_matcher.cpp"]; + node91 [label="tracer.cpp"]; + node92 [label="http_client.h"]; + node93 [label="default_http_client_curl.cpp"]; + node94 [label="clock.cpp"]; + node95 [label="sampling_priority.h"]; + node96 [label="collector_response.cpp"]; + node97 [label="hex.h"]; + node98 [label="limiter.cpp"]; + node99 [label="net_util.h"]; + node100 [label="error.cpp"]; + node101 [label="absl/strings/string_view.h"]; + node102 [label="trace_sampler_config.h"]; + node103 [label="tags.cpp"]; + node104 [label="cerr_logger.h"]; + node105 [label="trace_segment.cpp"]; + node106 [label="span_sampler_config.cpp"]; subgraph U { edge [dir=none]; } subgraph D { - node0 -> node20 []; - node1 -> node43 []; - node1 -> node68 []; - node1 -> node8 []; - node1 -> node24 []; - node1 -> node61 []; - node1 -> node11 []; - node1 -> node38 []; - node2 -> node7 []; - node2 -> node46 []; - node3 -> node30 []; - node4 -> node64 []; - node4 -> node23 []; + node0 -> node7 []; + node0 -> node47 []; + node1 -> node20 []; + node2 -> node44 []; + node2 -> node72 []; + node2 -> node8 []; + node2 -> node25 []; + node2 -> node65 []; + node2 -> node12 []; + node2 -> node39 []; + node3 -> node31 []; + node4 -> node68 []; + node4 -> node26 []; node4 -> node14 []; - node4 -> node77 []; - node5 -> node40 []; - node5 -> node48 []; - node5 -> node38 []; + node4 -> node82 []; + node5 -> node41 []; + node5 -> node49 []; + node5 -> node39 []; node6 -> node8 []; - node8 -> node95 []; - node9 -> node41 []; - node9 -> node81 []; - node9 -> node23 []; + node8 -> node101 []; + node9 -> node42 []; + node9 -> node88 []; + node9 -> node26 []; node9 -> node15 []; - node9 -> node35 []; - node10 -> node84 []; - node11 -> node8 []; - node12 -> node43 []; - node12 -> node64 []; - node12 -> node81 []; - node12 -> node25 []; - node12 -> node96 []; - node12 -> node14 []; - node12 -> node38 []; - node14 -> node77 []; - node14 -> node38 []; - node15 -> node41 []; + node9 -> node36 []; + node10 -> node89 []; + node11 -> node44 []; + node11 -> node68 []; + node11 -> node88 []; + node11 -> node24 []; + node11 -> node102 []; + node11 -> node14 []; + node11 -> node39 []; + node12 -> node8 []; + node14 -> node82 []; + node14 -> node39 []; + node15 -> node42 []; node15 -> node20 []; - node16 -> node77 []; - node17 -> node41 []; - node17 -> node72 []; - node17 -> node8 []; + node16 -> node82 []; + node17 -> node42 []; node17 -> node77 []; - node17 -> node38 []; - node18 -> node26 []; - node19 -> node42 []; - node19 -> node38 []; + node17 -> node8 []; + node17 -> node82 []; + node17 -> node39 []; + node18 -> node27 []; + node19 -> node43 []; + node19 -> node39 []; node20 -> node14 []; - node21 -> node20 []; - node22 -> node8 []; - node23 -> node26 []; - node23 -> node20 []; - node23 -> node77 []; - node24 -> node41 []; + node21 -> node97 []; + node22 -> node20 []; + node23 -> node8 []; node24 -> node14 []; + node24 -> node92 []; node24 -> node8 []; - node24 -> node77 []; + node25 -> node42 []; node25 -> node14 []; - node25 -> node88 []; node25 -> node8 []; - node27 -> node8 []; - node28 -> node61 []; - node28 -> node38 []; - node31 -> node21 []; - node32 -> node48 []; - node32 -> node25 []; - node32 -> node30 []; - node32 -> node67 []; - node32 -> node46 []; - node33 -> node4 []; - node33 -> node68 []; - node33 -> node6 []; + node25 -> node82 []; + node26 -> node27 []; + node26 -> node20 []; + node26 -> node82 []; + node28 -> node8 []; + node29 -> node65 []; + node29 -> node39 []; + node32 -> node22 []; + node33 -> node49 []; node33 -> node24 []; - node33 -> node77 []; - node33 -> node11 []; - node33 -> node8 []; - node33 -> node17 []; - node34 -> node39 []; - node36 -> node41 []; - node36 -> node56 []; - node36 -> node88 []; - node36 -> node39 []; - node37 -> node64 []; - node37 -> node7 []; - node38 -> node8 []; - node39 -> node35 []; - node39 -> node38 []; - node40 -> node14 []; - node40 -> node8 []; - node42 -> node8 []; - node43 -> node35 []; - node44 -> node88 []; - node44 -> node35 []; - node45 -> node43 []; - node45 -> node7 []; - node46 -> node39 []; - node47 -> node41 []; - node47 -> node72 []; - node47 -> node17 []; - node47 -> node77 []; - node47 -> node38 []; - node47 -> node14 []; - node47 -> node12 []; + node33 -> node31 []; + node33 -> node71 []; + node33 -> node47 []; + node34 -> node4 []; + node34 -> node72 []; + node34 -> node6 []; + node34 -> node25 []; + node34 -> node82 []; + node34 -> node12 []; + node34 -> node8 []; + node34 -> node17 []; + node35 -> node40 []; + node37 -> node42 []; + node37 -> node58 []; + node37 -> node92 []; + node37 -> node40 []; + node38 -> node68 []; + node38 -> node7 []; + node39 -> node8 []; + node40 -> node36 []; + node40 -> node39 []; + node41 -> node14 []; + node41 -> node8 []; + node43 -> node8 []; + node44 -> node36 []; + node45 -> node92 []; + node45 -> node36 []; + node46 -> node44 []; + node46 -> node7 []; + node47 -> node40 []; + node48 -> node42 []; + node48 -> node77 []; + node48 -> node17 []; + node48 -> node82 []; + node48 -> node39 []; node48 -> node14 []; - node48 -> node8 []; + node48 -> node11 []; node49 -> node14 []; - node50 -> node56 []; - node51 -> node72 []; - node52 -> node21 []; - node52 -> node62 []; - node52 -> node91 []; - node52 -> node60 []; - node52 -> node24 []; - node52 -> node23 []; - node52 -> node7 []; - node53 -> node23 []; - node54 -> node91 []; + node49 -> node8 []; + node50 -> node14 []; + node51 -> node58 []; + node52 -> node56 []; + node53 -> node77 []; + node54 -> node22 []; + node54 -> node66 []; + node54 -> node95 []; + node54 -> node63 []; + node54 -> node25 []; + node54 -> node26 []; + node54 -> node7 []; node55 -> node26 []; - node55 -> node21 []; - node55 -> node91 []; - node55 -> node7 []; - node55 -> node9 []; - node55 -> node24 []; - node56 -> node14 []; - node56 -> node77 []; - node56 -> node35 []; - node57 -> node42 []; - node57 -> node7 []; - node57 -> node8 []; - node57 -> node44 []; - node57 -> node6 []; - node57 -> node48 []; - node57 -> node13 []; - node57 -> node88 []; - node57 -> node84 []; - node58 -> node7 []; - node58 -> node78 []; - node59 -> node98 []; - node60 -> node20 []; + node56 -> node82 []; + node57 -> node95 []; + node58 -> node14 []; + node58 -> node82 []; + node58 -> node36 []; + node59 -> node27 []; + node59 -> node22 []; + node59 -> node95 []; + node59 -> node7 []; + node59 -> node9 []; + node59 -> node25 []; + node60 -> node43 []; + node60 -> node7 []; node60 -> node8 []; - node61 -> node14 []; - node61 -> node8 []; - node62 -> node41 []; - node62 -> node96 []; - node62 -> node20 []; - node62 -> node15 []; - node62 -> node77 []; - node62 -> node35 []; - node63 -> node68 []; - node64 -> node35 []; - node65 -> node6 []; - node66 -> node48 []; - node66 -> node96 []; - node66 -> node7 []; - node66 -> node67 []; - node67 -> node8 []; - node67 -> node77 []; - node67 -> node35 []; - node68 -> node41 []; - node68 -> node77 []; - node69 -> node42 []; - node69 -> node25 []; - node69 -> node60 []; - node69 -> node7 []; - node69 -> node80 []; - node69 -> node36 []; - node69 -> node62 []; + node60 -> node45 []; + node60 -> node6 []; + node60 -> node49 []; + node60 -> node13 []; + node60 -> node92 []; + node60 -> node89 []; + node61 -> node7 []; + node61 -> node83 []; + node62 -> node104 []; + node63 -> node20 []; + node63 -> node8 []; + node64 -> node56 []; + node64 -> node14 []; + node64 -> node82 []; + node65 -> node14 []; + node65 -> node8 []; + node66 -> node42 []; + node66 -> node102 []; + node66 -> node20 []; + node66 -> node15 []; + node66 -> node82 []; + node66 -> node36 []; + node67 -> node72 []; + node68 -> node36 []; node69 -> node6 []; - node69 -> node24 []; - node69 -> node61 []; - node71 -> node7 []; - node71 -> node67 []; - node73 -> node48 []; - node73 -> node98 []; + node70 -> node49 []; + node70 -> node102 []; + node70 -> node7 []; + node70 -> node71 []; + node71 -> node8 []; + node71 -> node82 []; + node71 -> node36 []; + node72 -> node42 []; + node72 -> node82 []; + node73 -> node43 []; + node73 -> node24 []; + node73 -> node63 []; node73 -> node7 []; - node73 -> node12 []; - node73 -> node67 []; - node73 -> node8 []; - node73 -> node78 []; - node73 -> node36 []; - node74 -> node92 []; - node74 -> node70 []; - node75 -> node79 []; - node75 -> node48 []; - node75 -> node38 []; - node76 -> node88 []; - node77 -> node29 []; - node78 -> node56 []; - node80 -> node8 []; - node81 -> node14 []; - node81 -> node20 []; - node81 -> node82 []; - node81 -> node77 []; - node81 -> node35 []; - node82 -> node14 []; - node82 -> node35 []; - node83 -> node80 []; - node84 -> node8 []; - node84 -> node77 []; - node85 -> node7 []; - node85 -> node24 []; - node85 -> node38 []; - node85 -> node82 []; - node85 -> node77 []; - node86 -> node4 []; - node86 -> node68 []; - node86 -> node47 []; - node86 -> node9 []; - node86 -> node36 []; - node86 -> node11 []; - node86 -> node40 []; - node86 -> node42 []; - node86 -> node62 []; - node86 -> node48 []; - node86 -> node7 []; - node86 -> node92 []; - node86 -> node24 []; - node86 -> node84 []; - node86 -> node67 []; - node86 -> node17 []; - node86 -> node80 []; - node87 -> node30 []; - node87 -> node44 []; + node73 -> node85 []; + node73 -> node37 []; + node73 -> node66 []; + node73 -> node6 []; + node73 -> node25 []; + node73 -> node65 []; + node74 -> node97 []; + node74 -> node89 []; + node74 -> node12 []; + node74 -> node64 []; + node74 -> node49 []; + node76 -> node7 []; + node76 -> node71 []; + node78 -> node49 []; + node78 -> node104 []; + node78 -> node7 []; + node78 -> node11 []; + node78 -> node71 []; + node78 -> node8 []; + node78 -> node83 []; + node78 -> node37 []; + node79 -> node99 []; + node79 -> node75 []; + node80 -> node84 []; + node80 -> node49 []; + node80 -> node39 []; + node81 -> node92 []; + node82 -> node30 []; + node83 -> node58 []; + node85 -> node8 []; + node86 -> node85 []; + node87 -> node14 []; + node87 -> node36 []; node88 -> node14 []; - node88 -> node38 []; - node88 -> node77 []; - node88 -> node35 []; - node89 -> node41 []; - node90 -> node60 []; - node91 -> node26 []; - node92 -> node77 []; - node93 -> node15 []; - node94 -> node38 []; - node96 -> node14 []; - node96 -> node20 []; - node96 -> node82 []; - node96 -> node77 []; - node96 -> node35 []; - node97 -> node48 []; - node97 -> node11 []; - node98 -> node42 []; - node99 -> node56 []; - node99 -> node60 []; - node99 -> node9 []; - node99 -> node24 []; - node99 -> node77 []; - node99 -> node40 []; - node99 -> node42 []; - node99 -> node4 []; - node99 -> node79 []; - node99 -> node62 []; - node99 -> node6 []; - node99 -> node11 []; - node99 -> node38 []; - node100 -> node42 []; - node100 -> node81 []; - node100 -> node7 []; - node100 -> node14 []; - node100 -> node67 []; + node88 -> node20 []; + node88 -> node87 []; + node88 -> node82 []; + node88 -> node36 []; + node89 -> node8 []; + node89 -> node82 []; + node90 -> node7 []; + node90 -> node25 []; + node90 -> node39 []; + node90 -> node87 []; + node90 -> node82 []; + node91 -> node4 []; + node91 -> node48 []; + node91 -> node9 []; + node91 -> node37 []; + node91 -> node12 []; + node91 -> node41 []; + node91 -> node43 []; + node91 -> node72 []; + node91 -> node56 []; + node91 -> node64 []; + node91 -> node66 []; + node91 -> node49 []; + node91 -> node7 []; + node91 -> node99 []; + node91 -> node25 []; + node91 -> node89 []; + node91 -> node71 []; + node91 -> node17 []; + node91 -> node85 []; + node92 -> node14 []; + node92 -> node39 []; + node92 -> node82 []; + node92 -> node36 []; + node93 -> node31 []; + node93 -> node45 []; + node94 -> node42 []; + node95 -> node27 []; + node96 -> node63 []; + node97 -> node84 []; + node98 -> node15 []; + node99 -> node82 []; + node100 -> node39 []; + node102 -> node14 []; + node102 -> node20 []; + node102 -> node87 []; + node102 -> node82 []; + node102 -> node36 []; + node103 -> node49 []; + node103 -> node12 []; + node104 -> node43 []; + node105 -> node4 []; + node105 -> node97 []; + node105 -> node58 []; + node105 -> node63 []; + node105 -> node9 []; + node105 -> node25 []; + node105 -> node82 []; + node105 -> node41 []; + node105 -> node43 []; + node105 -> node64 []; + node105 -> node66 []; + node105 -> node6 []; + node105 -> node12 []; + node105 -> node39 []; + node106 -> node43 []; + node106 -> node88 []; + node106 -> node7 []; + node106 -> node14 []; + node106 -> node71 []; } } diff --git a/doc/includes.svg b/doc/includes.svg index 5e55705b..11e0a44e 100644 --- a/doc/includes.svg +++ b/doc/includes.svg @@ -4,2170 +4,2296 @@ - - + + G - + node0 - -rate.cpp + +threaded_event_scheduler.cpp - - -node20 - -rate.h + + +node7 + +json.hpp - + -node0->node20 - - +node0->node7 + + + + + +node47 + +threaded_event_scheduler.h + + + +node0->node47 + + node1 - -span_data.cpp - - - -node8 - -string_view.h + +rate.cpp - - -node1->node8 - - - - - -node11 - -tags.h + + +node20 + +rate.h - - -node1->node11 - - + + +node1->node20 + + - - -node24 - -span_data.h + + +node2 + +span_data.cpp - - -node1->node24 - - + + +node8 + +string_view.h - - -node38 - -error.h + + +node2->node8 + + - - -node1->node38 - - + + +node12 + +tags.h - - -node43 - -span_defaults.h + + +node2->node12 + + - - -node1->node43 - - + + +node25 + +span_data.h - - -node61 - -msgpack.h + + +node2->node25 + + - - -node1->node61 - - + + +node39 + +error.h - - -node68 - -span_config.h + + +node2->node39 + + - - -node1->node68 - - + + +node44 + +span_defaults.h - - -node2 - -threaded_event_scheduler.cpp + + +node2->node44 + + - - -node7 - -json.hpp + + +node65 + +msgpack.h - - -node2->node7 - - + + +node2->node65 + + - - -node46 - -threaded_event_scheduler.h + + +node72 + +span_config.h - - -node2->node46 - - + + +node2->node72 + + node3 - -default_http_client_null.cpp + +default_http_client_null.cpp - - -node30 - -default_http_client.h + + +node31 + +default_http_client.h - + -node3->node30 - - +node3->node31 + + node4 - -trace_segment.h + +trace_segment.h node14 - -expected.h + +expected.h node4->node14 - - + + - - -node23 - -sampling_decision.h + + +node26 + +sampling_decision.h - + -node4->node23 - - +node4->node26 + + - - -node64 - -propagation_style.h + + +node68 + +propagation_style.h - + -node4->node64 - - +node4->node68 + + - - -node77 - -optional.h + + +node82 + +optional.h - + -node4->node77 - - +node4->node82 + + node5 - -tag_propagation.cpp + +tag_propagation.cpp - + -node5->node38 - - +node5->node39 + + - - -node40 - -tag_propagation.h + + +node41 + +tag_propagation.h - + -node5->node40 - - +node5->node41 + + - - -node48 - -parse_util.h + + +node49 + +parse_util.h - + -node5->node48 - - +node5->node49 + + node6 - -dict_writer.h + +dict_writer.h node6->node8 - - + + - - -node95 - -absl/strings/string_view.h + + +node101 + +absl/strings/string_view.h - + -node8->node95 - - +node8->node101 + + node9 - -span_sampler.h + +span_sampler.h node15 - -limiter.h + +limiter.h node9->node15 - - + + - + -node9->node23 - - +node9->node26 + + - - -node35 - -json_fwd.hpp + + +node36 + +json_fwd.hpp - + -node9->node35 - - +node9->node36 + + - - -node41 - -clock.h + + +node42 + +clock.h - + -node9->node41 - - +node9->node42 + + - - -node81 - -span_sampler_config.h + + +node88 + +span_sampler_config.h - + -node9->node81 - - +node9->node88 + + node10 - -dict_reader.cpp + +dict_reader.cpp - - -node84 - -dict_reader.h + + +node89 + +dict_reader.h - + -node10->node84 - - +node10->node89 + + - - -node11->node8 - - + + +node11 + +tracer_config.h - - -node12 - -tracer_config.h + + +node11->node14 + + - - -node12->node14 - - + + +node24 + +datadog_agent_config.h - - -node25 - -datadog_agent_config.h + + +node11->node24 + + - - -node12->node25 - - + + +node11->node39 + + - - -node12->node38 - - + + +node11->node44 + + - + -node12->node43 - - +node11->node68 + + - + -node12->node64 - - +node11->node88 + + - - -node12->node81 - - + + +node102 + +trace_sampler_config.h - - -node96 - -trace_sampler_config.h + + +node11->node102 + + - - -node12->node96 - - + + +node12->node8 + + node13 - -curl/curl.h + +curl/curl.h - + -node14->node38 - - +node14->node39 + + - + -node14->node77 - - +node14->node82 + + node15->node20 - - + + - + -node15->node41 - - +node15->node42 + + node16 - -optional.cpp + +optional.cpp - + -node16->node77 - - +node16->node82 + + node17 - -span.h + +span.h node17->node8 - - + + - + -node17->node38 - - +node17->node39 + + - + -node17->node41 - - +node17->node42 + + - - -node72 - -id_generator.h + + +node77 + +id_generator.h - + -node17->node72 - - +node17->node77 + + - + -node17->node77 - - +node17->node82 + + node18 - -sampling_mechanism.cpp + +sampling_mechanism.cpp - - -node26 - -sampling_mechanism.h + + +node27 + +sampling_mechanism.h - + -node18->node26 - - +node18->node27 + + node19 - -logger.cpp + +logger.cpp - + -node19->node38 - - +node19->node39 + + - - -node42 - -logger.h + + +node43 + +logger.h - + -node19->node42 - - +node19->node43 + + node20->node14 - - + + node21 - -sampling_util.h + +hex.cpp - + + +node97 + +hex.h + + -node21->node20 - - +node21->node97 + + node22 - -string_view.cpp + +sampling_util.h - + -node22->node8 - - +node22->node20 + + - - -node23->node20 - - + + +node23 + +string_view.cpp - + -node23->node26 - - - - - -node23->node77 - - +node23->node8 + + - + node24->node8 - - + + - + node24->node14 - - + + - - -node24->node41 - - + + +node92 + +http_client.h - - -node24->node77 - - + + +node24->node92 + + - + node25->node8 - - + + - + node25->node14 - - + + - - -node88 - -http_client.h + + +node25->node42 + + - - -node25->node88 - - + + +node25->node82 + + - - -node27 - -glob.h + + +node26->node20 + + + + + +node26->node27 + + - + -node27->node8 - - +node26->node82 + + node28 - -msgpack.cpp + +glob.h - - -node28->node38 - - - - + -node28->node61 - - +node28->node8 + + node29 - -absl/types/optional.h - - - -node31 - -sampling_util.cpp + +msgpack.cpp - + -node31->node21 - - +node29->node39 + + + + + +node29->node65 + + + + + +node30 + +absl/types/optional.h node32 - -datadog_agent_config.cpp + +sampling_util.cpp - - -node32->node25 - - + + +node32->node22 + + + + + +node33 + +datadog_agent_config.cpp - + -node32->node30 - - +node33->node24 + + - - -node32->node46 - - - - - -node32->node48 - - - - - -node67 - -environment.h - - + -node32->node67 - - - - - -node33 - -span.cpp +node33->node31 + + - + -node33->node4 - - +node33->node47 + + - - -node33->node6 - - - - - -node33->node8 - - + + +node33->node49 + + - - -node33->node11 - - + + +node71 + +environment.h - - -node33->node17 - - + + +node33->node71 + + - - -node33->node24 - - + + +node34 + +span.cpp - + -node33->node68 - - +node34->node4 + + - - -node33->node77 - - + + +node34->node6 + + - - -node34 - -event_scheduler.cpp + + +node34->node8 + + - - -node39 - -event_scheduler.h + + +node34->node12 + + - + -node34->node39 - - +node34->node17 + + - - -node36 - -datadog_agent.h + + +node34->node25 + + - - -node36->node39 - - + + +node34->node72 + + - - -node36->node41 - - + + +node34->node82 + + - - -node56 - -collector.h + + +node35 + +event_scheduler.cpp - - -node36->node56 - - + + +node40 + +event_scheduler.h - - -node36->node88 - - + + +node35->node40 + + node37 - -propagation_style.cpp - - - -node37->node7 - - + +datadog_agent.h - + -node37->node64 - - +node37->node40 + + - + + +node37->node42 + + + + + +node58 + +collector.h + + + +node37->node58 + + + + + +node37->node92 + + + + + +node38 + +propagation_style.cpp + + -node38->node8 - - +node38->node7 + + + + + +node38->node68 + + - + -node39->node35 - - +node39->node8 + + - + -node39->node38 - - - - - -node40->node8 - - +node40->node36 + + - + -node40->node14 - - +node40->node39 + + - + -node42->node8 - - - - - -node43->node35 - - +node41->node8 + + - - -node44 - -curl.h + + +node41->node14 + + - - -node44->node35 - - + + +node43->node8 + + - + -node44->node88 - - +node44->node36 + + node45 - -span_defaults.cpp - - - -node45->node7 - - + +curl.h - + -node45->node43 - - +node45->node36 + + - - -node46->node39 - - + + +node45->node92 + + - - -node47 - -tracer.h + + +node46 + +span_defaults.cpp - - -node47->node12 - - + + +node46->node7 + + - - -node47->node14 - - + + +node46->node44 + + - - -node47->node17 - - + + +node47->node40 + + - - -node47->node38 - - + + +node48 + +tracer.h - - -node47->node41 - - + + +node48->node11 + + - - -node47->node72 - - + + +node48->node14 + + - + -node47->node77 - - +node48->node17 + + - - -node48->node8 - - + + +node48->node39 + + - - -node48->node14 - - + + +node48->node42 + + - - -node49 - -expected.cpp + + +node48->node77 + + - + + +node48->node82 + + + + +node49->node8 + + + + + node49->node14 - - + + node50 - -collector.cpp + +expected.cpp - + -node50->node56 - - +node50->node14 + + node51 - -id_generator.cpp + +collector.cpp - + -node51->node72 - - +node51->node58 + + node52 - -trace_sampler.cpp + +extracted_data.cpp - - -node52->node7 - - + + +node56 + +extracted_data.h - + -node52->node21 - - - - - -node52->node23 - - - - - -node52->node24 - - - - - -node60 - -collector_response.h - - - -node52->node60 - - - - - -node62 - -trace_sampler.h - - - -node52->node62 - - - - - -node91 - -sampling_priority.h - - - -node52->node91 - - +node52->node56 + + node53 - -sampling_decision.cpp + +id_generator.cpp - - -node53->node23 - - + + +node53->node77 + + node54 - -sampling_priority.cpp + +trace_sampler.cpp - + -node54->node91 - - +node54->node7 + + - - -node55 - -span_sampler.cpp + + +node54->node22 + + - - -node55->node7 - - + + +node54->node25 + + - - -node55->node9 - - + + +node54->node26 + + - - -node55->node21 - - + + +node63 + +collector_response.h - - -node55->node24 - - + + +node54->node63 + + + + + +node66 + +trace_sampler.h + + + +node54->node66 + + + + + +node95 + +sampling_priority.h + + + +node54->node95 + + + + + +node55 + +sampling_decision.cpp node55->node26 - - + + + + + +node56->node82 + + + + + +node57 + +sampling_priority.cpp - + -node55->node91 - - +node57->node95 + + - - -node56->node14 - - + + +node58->node14 + + - - -node56->node35 - - + + +node58->node36 + + - - -node56->node77 - - + + +node58->node82 + + - - -node57 - -curl.cpp + + +node59 + +span_sampler.cpp - - -node57->node6 - - + + +node59->node7 + + - + -node57->node7 - - +node59->node9 + + + + + +node59->node22 + + - + -node57->node8 - - +node59->node25 + + - - -node57->node13 - - + + +node59->node27 + + - - -node57->node42 - - + + +node59->node95 + + - - -node57->node44 - - + + +node60 + +curl.cpp + + + +node60->node6 + + + + + +node60->node7 + + - + -node57->node48 - - +node60->node8 + + - - -node57->node84 - - + + +node60->node13 + + - - -node57->node88 - - + + +node60->node43 + + - - -node58 - -null_collector.cpp + + +node60->node45 + + - - -node58->node7 - - + + +node60->node49 + + - - -node78 - -null_collector.h + + +node60->node89 + + - + -node58->node78 - - +node60->node92 + + - - -node59 - -cerr_logger.cpp + + +node61 + +null_collector.cpp - - -node98 - -cerr_logger.h + + +node61->node7 + + - - -node59->node98 - - + + +node83 + +null_collector.h - + -node60->node8 - - +node61->node83 + + - - -node60->node20 - - + + +node62 + +cerr_logger.cpp - - -node61->node8 - - + + +node104 + +cerr_logger.h - + -node61->node14 - - +node62->node104 + + - - -node62->node15 - - - - - -node62->node20 - - + + +node63->node8 + + - - -node62->node35 - - + + +node63->node20 + + - - -node62->node41 - - + + +node64 + +w3c_propagation.h - - -node62->node77 - - + + +node64->node14 + + - + -node62->node96 - - +node64->node56 + + - - -node63 - -span_config.cpp + + +node64->node82 + + - - -node63->node68 - - + + +node65->node8 + + - - -node64->node35 - - + + +node65->node14 + + - - -node65 - -dict_writer.cpp + + +node66->node15 + + - + -node65->node6 - - - - - -node66 - -trace_sampler_config.cpp +node66->node20 + + - + -node66->node7 - - +node66->node36 + + - - -node66->node48 - - - - - -node66->node67 - - + + +node66->node42 + + - + -node66->node96 - - - - - -node67->node8 - - +node66->node82 + + - - -node67->node35 - - + + +node66->node102 + + - - -node67->node77 - - + + +node67 + +span_config.cpp - - -node68->node41 - - + + +node67->node72 + + - - -node68->node77 - - + + +node68->node36 + + node69 - -datadog_agent.cpp + +dict_writer.cpp - + node69->node6 - - - - - -node69->node7 - - + + - - -node69->node24 - - + + +node70 + +trace_sampler_config.cpp - - -node69->node25 - - + + +node70->node7 + + - - -node69->node36 - - + + +node70->node49 + + - + -node69->node42 - - - - - -node69->node60 - - - - - -node69->node61 - - - - - -node69->node62 - - +node70->node71 + + - - -node80 - -version.h + + +node70->node102 + + - - -node69->node80 - - + + +node71->node8 + + - - -node70 - -winsock.h + + +node71->node36 + + - - -node71 - -environment.cpp + + +node71->node82 + + - - -node71->node7 - - + + +node72->node42 + + - - -node71->node67 - - + + +node72->node82 + + node73 - -tracer_config.cpp + +datadog_agent.cpp + + + +node73->node6 + + - + node73->node7 - - + + - - -node73->node8 - - + + +node73->node24 + + - - -node73->node12 - - + + +node73->node25 + + - - -node73->node36 - - + + +node73->node37 + + - - -node73->node48 - - + + +node73->node43 + + - - -node73->node67 - - + + +node73->node63 + + - - -node73->node78 - - + + +node73->node65 + + - - -node73->node98 - - + + +node73->node66 + + + + + +node85 + +version.h + + + +node73->node85 + + node74 - -net_util.cpp - - - -node74->node70 - - + +w3c_propagation.cpp - - -node92 - -net_util.h + + +node74->node12 + + - + -node74->node92 - - +node74->node49 + + + + + +node74->node64 + + + + + +node74->node89 + + + + + +node74->node97 + + node75 - -parse_util.cpp - - - -node75->node38 - - + +winsock.h - - -node75->node48 - - + + +node76 + +environment.cpp - - -node79 - -charconv + + +node76->node7 + + - + -node75->node79 - - +node76->node71 + + - - -node76 - -http_client.cpp + + +node78 + +tracer_config.cpp - + -node76->node88 - - +node78->node7 + + + + + +node78->node8 + + - + -node77->node29 - - +node78->node11 + + - - -node78->node56 - - + + +node78->node37 + + - - -node80->node8 - - + + +node78->node49 + + - + + +node78->node71 + + + + -node81->node14 - - +node78->node83 + + - - -node81->node20 - - + + +node78->node104 + + - - -node81->node35 - - + + +node79 + +net_util.cpp - + -node81->node77 - - +node79->node75 + + - - -node82 - -span_matcher.h + + +node99 + +net_util.h - + -node81->node82 - - +node79->node99 + + - - -node82->node14 - - + + +node80 + +parse_util.cpp - + -node82->node35 - - +node80->node39 + + - - -node83 - -version.cpp + + +node80->node49 + + + + + +node84 + +charconv + + + +node80->node84 + + - + + +node81 + +http_client.cpp + + -node83->node80 - - +node81->node92 + + - + -node84->node8 - - +node82->node30 + + - + -node84->node77 - - - - - -node85 - -span_matcher.cpp +node83->node58 + + - + -node85->node7 - - +node85->node8 + + - + + +node86 + +version.cpp + + -node85->node24 - - +node86->node85 + + - - -node85->node38 - - + + +node87 + +span_matcher.h - - -node85->node77 - - + + +node87->node14 + + - + -node85->node82 - - +node87->node36 + + - - -node86 - -tracer.cpp + + +node88->node14 + + - + -node86->node4 - - +node88->node20 + + - - -node86->node7 - - - - + -node86->node9 - - - - - -node86->node11 - - +node88->node36 + + - - -node86->node17 - - + + +node88->node82 + + - - -node86->node24 - - + + +node88->node87 + + - + -node86->node36 - - +node89->node8 + + - - -node86->node40 - - + + +node89->node82 + + - - -node86->node42 - - + + +node90 + +span_matcher.cpp - - -node86->node47 - - + + +node90->node7 + + - - -node86->node48 - - + + +node90->node25 + + - + -node86->node62 - - - - - -node86->node67 - - +node90->node39 + + - - -node86->node68 - - + + +node90->node82 + + - - -node86->node80 - - + + +node90->node87 + + - - -node86->node84 - - + + +node91 + +tracer.cpp - + -node86->node92 - - - - - -node87 - -default_http_client_curl.cpp +node91->node4 + + - - -node87->node30 - - + + +node91->node7 + + - - -node87->node44 - - + + +node91->node9 + + - - -node88->node14 - - + + +node91->node12 + + - - -node88->node35 - - + + +node91->node17 + + - - -node88->node38 - - + + +node91->node25 + + - - -node88->node77 - - + + +node91->node37 + + - - -node89 - -clock.cpp + + +node91->node41 + + - - -node89->node41 - - + + +node91->node43 + + - - -node90 - -collector_response.cpp + + +node91->node48 + + - - -node90->node60 - - + + +node91->node49 + + - - -node91->node26 - - + + +node91->node56 + + - - -node92->node77 - - + + +node91->node64 + + - - -node93 - -limiter.cpp + + +node91->node66 + + - + -node93->node15 - - - - - -node94 - -error.cpp +node91->node71 + + - - -node94->node38 - - + + +node91->node72 + + - + -node96->node14 - - +node91->node85 + + - + + +node91->node89 + + + + + +node91->node99 + + + + -node96->node20 - - +node92->node14 + + - + -node96->node35 - - +node92->node36 + + - + + +node92->node39 + + + + -node96->node77 - - +node92->node82 + + - - -node96->node82 - - + + +node93 + +default_http_client_curl.cpp - - -node97 - -tags.cpp + + +node93->node31 + + - + -node97->node11 - - +node93->node45 + + - - -node97->node48 - - + + +node94 + +clock.cpp - + -node98->node42 - - +node94->node42 + + - - -node99 - -trace_segment.cpp + + +node95->node27 + + - - -node99->node4 - - + + +node96 + +collector_response.cpp - - -node99->node6 - - + + +node96->node63 + + - + -node99->node9 - - +node97->node84 + + - - -node99->node11 - - + + +node98 + +limiter.cpp - + -node99->node24 - - +node98->node15 + + - - -node99->node38 - - + + +node99->node82 + + + + + +node100 + +error.cpp - + -node99->node40 - - +node100->node39 + + - + -node99->node42 - - +node102->node14 + + - - -node99->node56 - - + + +node102->node20 + + - - -node99->node60 - - + + +node102->node36 + + - + -node99->node62 - - - - - -node99->node77 - - +node102->node82 + + - + -node99->node79 - - +node102->node87 + + - - -node100 - -span_sampler_config.cpp + + +node103 + +tags.cpp - - -node100->node7 - - + + +node103->node12 + + - - -node100->node14 - - + + +node103->node49 + + - + -node100->node42 - - +node104->node43 + + - - -node100->node67 - - + + +node105 + +trace_segment.cpp - + -node100->node81 - - +node105->node4 + + + + + +node105->node6 + + + + + +node105->node9 + + + + + +node105->node12 + + + + + +node105->node25 + + + + + +node105->node39 + + + + + +node105->node41 + + + + + +node105->node43 + + + + + +node105->node58 + + + + + +node105->node63 + + + + + +node105->node64 + + + + + +node105->node66 + + + + + +node105->node82 + + + + + +node105->node97 + + + + + +node106 + +span_sampler_config.cpp + + + +node106->node7 + + + + + +node106->node14 + + + + + +node106->node43 + + + + + +node106->node71 + + + + + +node106->node88 + + From b3c9fa454c8beb6270504e1d6d3ad28d94b28c76 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 01:42:36 -0500 Subject: [PATCH 19/54] TODO: AMEND: add a fuzzer for W3C propagation --- CMakeLists.txt | 11 ++++ bin/format | 4 +- fuzz/w3c-propagation/CMakeLists.txt | 3 + fuzz/w3c-propagation/fuzz.cpp | 86 +++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 fuzz/w3c-propagation/CMakeLists.txt create mode 100644 fuzz/w3c-propagation/fuzz.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c93261e..30825d1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ option(BUILD_COVERAGE "Build code with code coverage profiling instrumentation" option(BUILD_EXAMPLE "Build the example program (example/)" OFF) option(BUILD_TESTING "Build the unit tests (test/)" OFF) option(SANITIZE "Build with address sanitizer and undefined behavior sanitizer" OFF) +option(FUZZ_W3C_PROPAGATION "Build a fuzzer for W3C propagation" OFF) set(CMAKE_BUILD_TYPE "RelWithDebInfo") set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -20,6 +21,16 @@ include(ProcessorCount) ProcessorCount(NUM_PROCESSORS) set(MAKE_JOB_COUNT ${NUM_PROCESSORS} CACHE STRING "Number of jobs to use when building libcurl") +if(FUZZ_W3C_PROPAGATION) + add_compile_options(-fsanitize=fuzzer) + add_link_options(-fsanitize=fuzzer) + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) + add_compile_options(-fsanitize=undefined) + add_link_options(-fsanitize=undefined) + add_subdirectory(fuzz/w3c-propagation) +endif() + if (SANITIZE) add_compile_options(-fsanitize=address) add_link_options(-fsanitize=address) diff --git a/bin/format b/bin/format index 5efa9fd9..3c8e97b3 100755 --- a/bin/format +++ b/bin/format @@ -12,7 +12,7 @@ formatter=clang-format-$version formatter_options="--style=file -i $@" find_sources() { - find src/ example/ test/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" + find src/ example/ test/ fuzz/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" } # If the correct version of clang-format is installed, then use it and quit. @@ -35,7 +35,7 @@ if [ "$(docker image ls --quiet $image | wc -l)" -eq 0 ]; then case "$response" in y|Y|yes|Yes|YES|'') ;; *) exit ;; - esac + esac fi mount_path=/mnt/host diff --git a/fuzz/w3c-propagation/CMakeLists.txt b/fuzz/w3c-propagation/CMakeLists.txt new file mode 100644 index 00000000..45c62d40 --- /dev/null +++ b/fuzz/w3c-propagation/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(fuzz fuzz.cpp) + +target_link_libraries(fuzz dd_trace_cpp) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp new file mode 100644 index 00000000..97dcd1fc --- /dev/null +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace dd = datadog::tracing; + +namespace { + +dd::Tracer& tracer_singleton() { + static auto tracer = []() { + dd::TracerConfig config; + config.defaults.service = "fuzzer"; + config.collector = std::make_shared(); + + const auto finalized_config = dd::finalize_config(config); + if (!finalized_config) { + std::abort(); + } + + return dd::Tracer{*finalized_config}; + }(); + + return tracer; +} + +struct MockDictReader : public dd::DictReader { + dd::StringView traceparent; + dd::StringView tracestate; + + dd::Optional lookup(dd::StringView key) const override { + if (key == "traceparent") { + return traceparent; + } + if (key == "tracestate") { + return tracestate; + } + return dd::nullopt; + } + + void visit( + const std::function& + visitor) const override { + visitor("traceparent", traceparent); + visitor("tracestate", tracestate); + } +}; + +struct MockDictWriter : public dd::DictWriter { + void set(dd::StringView, dd::StringView) override {} +}; + +} // namespace + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, size_t size) { + auto& tracer = tracer_singleton(); + + const auto begin_traceparent = reinterpret_cast(data); + const auto end = begin_traceparent + size; + for (const char* begin_tracestate = begin_traceparent; + begin_tracestate != end; ++begin_tracestate) { + MockDictReader reader; + reader.traceparent = dd::range(begin_traceparent, begin_tracestate); + reader.tracestate = dd::range(begin_tracestate, end); + + const auto span = tracer.extract_span(reader); + if (!span) { + continue; + } + + MockDictWriter writer; + span->inject(writer); + } + + return 0; +} From 065945c71e009e03f202db04f427ab15d8e4d815 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 12:04:12 -0500 Subject: [PATCH 20/54] TODO: AMEND: thread_local, not that it matters --- fuzz/w3c-propagation/fuzz.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp index 97dcd1fc..d6262700 100644 --- a/fuzz/w3c-propagation/fuzz.cpp +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -18,7 +18,7 @@ namespace dd = datadog::tracing; namespace { dd::Tracer& tracer_singleton() { - static auto tracer = []() { + thread_local auto tracer = []() { dd::TracerConfig config; config.defaults.service = "fuzzer"; config.collector = std::make_shared(); From 3bbf910ddfb1c142fa5d061e32f9fea116dd102c Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 13:34:10 -0500 Subject: [PATCH 21/54] TODO: AMEND: document the fuzzer --- CMakeLists.txt | 16 +++++++-------- fuzz/README.md | 26 ++++++++++++++++++++++++ fuzz/w3c-propagation/README.md | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 fuzz/README.md create mode 100644 fuzz/w3c-propagation/README.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 30825d1b..0f2442da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,22 +21,22 @@ include(ProcessorCount) ProcessorCount(NUM_PROCESSORS) set(MAKE_JOB_COUNT ${NUM_PROCESSORS} CACHE STRING "Number of jobs to use when building libcurl") -if(FUZZ_W3C_PROPAGATION) - add_compile_options(-fsanitize=fuzzer) - add_link_options(-fsanitize=fuzzer) +function(add_sanitizers) add_compile_options(-fsanitize=address) add_link_options(-fsanitize=address) add_compile_options(-fsanitize=undefined) add_link_options(-fsanitize=undefined) +endfunction() + +if(FUZZ_W3C_PROPAGATION) + add_compile_options(-fsanitize=fuzzer) + add_link_options(-fsanitize=fuzzer) + add_sanitizers() add_subdirectory(fuzz/w3c-propagation) endif() if (SANITIZE) - add_compile_options(-fsanitize=address) - add_link_options(-fsanitize=address) - add_compile_options(-fsanitize=undefined) - add_link_options(-fsanitize=undefined) - # Thread sanitizer fails with "unexpected memory mapping". + add_sanitizers() endif() include (ExternalProject) diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 00000000..3f0181c3 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,26 @@ +Fuzzers +======= +Each subdirectory here contains the source of an executable that [fuzz tests][1] +some part of the library using [LLVM's libfuzzer][2]. + +There is a toplevel CMake boolean option associated with each fuzzer. The naming +convention is `FUZZ_`, e.g. +`FUZZ_W3C_PROPAGATION` for the fuzzer defined in +[fuzz/w3c-propagation/](./w3c-propagation/). The resulting binary is called +`fuzz` by convention. + +When building a fuzzer, the toolchain must be clang-based. For example, this +is how to build the fuzzer in [fuzz/w3c-propagation](./w3c-propagation/) from +the root of the repository: +```console +$ rm -rf .build && mkdir .build # if toolchain or test setup need clearing +$ cd .build +$ CC=clang CXX=clang++ cmake .. -DFUZZ_W3C_PROPAGATION=ON +$ make -j $(nproc) +$ fuzz/w3c-propagation/fuzz + +[... fuzzer output ...] +``` + +[1]: https://en.wikipedia.org/wiki/Fuzzing +[2]: https://llvm.org/docs/LibFuzzer.html diff --git a/fuzz/w3c-propagation/README.md b/fuzz/w3c-propagation/README.md new file mode 100644 index 00000000..ea0edf79 --- /dev/null +++ b/fuzz/w3c-propagation/README.md @@ -0,0 +1,36 @@ +W3C Propagation Fuzzer +====================== +This directory defines an executable, `fuzz`, that fuzz tests extraction and +injection of the W3C tracing HTTP headers "traceparent" and "tracestate". + +Libfuzzer invokes the `LLVMFuzzerTestOneInput` function repeatedly with a binary +blob of varying size and contents. For each blob, [fuzz.cpp](./fuzz.cpp) runs +its test multiple times. The input blob is interpreted in the following way: +```text +blob: _ _ _ _ _ _ _ _ _ _ + +iteration 0: s s s s s s s s s + +iteration 1: p s s s s s s s s + +iteration 2: p p s s s s s s s + +iteration 3: p p p s s s s s s + +... + +iteration 8: p p p p p p p p s + +iteration 9: p p p p p p p p p +``` + The `p`-labeled bytes are the "traceparent" header value, while the `s`-labeled + bytes are the "tracestate" header value. So, for an input blob of length `n`, + the `LLVMFuzzerTestOneInput` runs its test `n + 1` times, for the `n + 1` + possible divisions of the input between "traceparent" and "tracestate". + + Each test uses a singleton[^1] `Tracer` to `extract_span` from a `DictReader` + containing the "traceparent" and "tracestate" headers. If that succeeds, then + the test `inject`s the resulting span into a no-op `DictWriter`. + +[^1]: thread-local, actually, though it doesn't matter because even libfuzzer's + "worker" mode forks instead of threads \ No newline at end of file From dcf6cda4cc34d21f733d49af4826381e6e08896f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 15:21:44 -0500 Subject: [PATCH 22/54] TODO: AMEND: typo in readme --- fuzz/w3c-propagation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz/w3c-propagation/README.md b/fuzz/w3c-propagation/README.md index ea0edf79..f803c278 100644 --- a/fuzz/w3c-propagation/README.md +++ b/fuzz/w3c-propagation/README.md @@ -7,7 +7,7 @@ Libfuzzer invokes the `LLVMFuzzerTestOneInput` function repeatedly with a binary blob of varying size and contents. For each blob, [fuzz.cpp](./fuzz.cpp) runs its test multiple times. The input blob is interpreted in the following way: ```text -blob: _ _ _ _ _ _ _ _ _ _ +blob: _ _ _ _ _ _ _ _ _ iteration 0: s s s s s s s s s @@ -33,4 +33,4 @@ iteration 9: p p p p p p p p p the test `inject`s the resulting span into a no-op `DictWriter`. [^1]: thread-local, actually, though it doesn't matter because even libfuzzer's - "worker" mode forks instead of threads \ No newline at end of file + "worker" mode forks instead of threads From 9a9212c05d2b2bd8a095f7f6f21b034c7cde8ce6 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 15:29:34 -0500 Subject: [PATCH 23/54] TODO: AMEND: be honest --- fuzz/w3c-propagation/fuzz.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp index d6262700..f02323d6 100644 --- a/fuzz/w3c-propagation/fuzz.cpp +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -68,7 +68,7 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, size_t size) { const auto begin_traceparent = reinterpret_cast(data); const auto end = begin_traceparent + size; for (const char* begin_tracestate = begin_traceparent; - begin_tracestate != end; ++begin_tracestate) { + begin_tracestate <= end; ++begin_tracestate) { MockDictReader reader; reader.traceparent = dd::range(begin_traceparent, begin_tracestate); reader.tracestate = dd::range(begin_tracestate, end); From e923b03087c5d72203d9ee6777f3e7eee48bd691 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 17:46:07 -0500 Subject: [PATCH 24/54] shellcheck --- bin/bazel-build | 2 +- bin/cmake-build | 4 ++-- bin/example | 7 ++++--- bin/format | 8 ++++++-- bin/install-cmake | 2 ++ bin/test | 1 + 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bin/bazel-build b/bin/bazel-build index d46c18e6..4c295c0d 100755 --- a/bin/bazel-build +++ b/bin/bazel-build @@ -2,6 +2,6 @@ set -e -cd $(dirname "$0")/.. +cd "$(dirname "$0")"/.. bazelisk build dd_trace_cpp diff --git a/bin/cmake-build b/bin/cmake-build index 0cd84cf7..1ba61e27 100755 --- a/bin/cmake-build +++ b/bin/cmake-build @@ -2,9 +2,9 @@ set -e -cd $(dirname "$0")/.. +cd "$(dirname "$0")"/.. mkdir -p .build cd .build cmake .. -make -j $(nproc) +make -j "$(nproc)" diff --git a/bin/example b/bin/example index 5139ca41..21dfe2c1 100755 --- a/bin/example +++ b/bin/example @@ -19,8 +19,9 @@ cd "$(dirname "$0")"/.. mkdir -p .build cd .build -cmake .. $coverage_flags -DBUILD_EXAMPLE=1 -make -j $(nproc) $verbosity_flags +cmake .. -DBUILD_EXAMPLE=1 +# shellcheck disable=SC2086 +make -j "$(nproc)" $verbosity_flags if [ "$build_only" -eq 1 ]; then exit @@ -35,6 +36,6 @@ if [ "$DD_API_KEY" = '' ]; then fi docker compose --project-directory ../example up --detach --remove-orphans # docker compose --project-directory ../example logs --follow & -./example/example"$@" +./example/example "$@" docker compose --project-directory ../example down # wait diff --git a/bin/format b/bin/format index 3c8e97b3..588e9230 100755 --- a/bin/format +++ b/bin/format @@ -1,5 +1,7 @@ #!/bin/sh +set -e + # Go to the repository root directory. cd "$(dirname "$0")"/.. @@ -9,7 +11,7 @@ cd "$(dirname "$0")"/.. # occasionally bumps the required version, reformatting everything. version=14 formatter=clang-format-$version -formatter_options="--style=file -i $@" +formatter_options="--style=file -i" find_sources() { find src/ example/ test/ fuzz/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" @@ -17,6 +19,7 @@ find_sources() { # If the correct version of clang-format is installed, then use it and quit. if >/dev/null command -v "$formatter"; then + # shellcheck disable=SC2086 find_sources -print0 | xargs -0 "$formatter" $formatter_options exit fi @@ -31,7 +34,7 @@ fi image=silkeh/clang:$version if [ "$(docker image ls --quiet $image | wc -l)" -eq 0 ]; then printf '%s is not installed. Download docker image %s instead? [Y/n] ' "$formatter" "$image" - read response + read -r response case "$response" in y|Y|yes|Yes|YES|'') ;; *) exit ;; @@ -52,6 +55,7 @@ process_arg() { } docker_clang_format() { + # shellcheck disable=SC2086 find_sources | while IFS= read -r arg; do process_arg "$arg" done | xargs -0 \ diff --git a/bin/install-cmake b/bin/install-cmake index 66761dcd..26dadba9 100755 --- a/bin/install-cmake +++ b/bin/install-cmake @@ -1,5 +1,7 @@ #!/bin/sh +set -e + # Install a recent binary release of cmake. # Kitware produces a self-installing tarball. diff --git a/bin/test b/bin/test index 79c3afd0..f7cddf38 100755 --- a/bin/test +++ b/bin/test @@ -37,6 +37,7 @@ cd "$(dirname "$0")"/.. mkdir -p .build cd .build +# shellcheck disable=SC2086 cmake .. $coverage_flags -DBUILD_TESTING=1 # Clean up any code coverage artifacts. find . -type f -name '*.gcda' -print0 | xargs -0 rm -f From ac95cd1bc60f66520a3f95646025fb37a18efaf4 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 17:51:28 -0500 Subject: [PATCH 25/54] shellcheck --- .circleci/config.yml | 9 +++++++++ Dockerfile | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fe7404c..82dd2a21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,14 @@ jobs: - checkout - run: bin/check-format + shellcheck: + docker: + - image: "datadog/docker-library:dd-trace-cpp-ci" + resource_class: small + steps: + - checkout + - run: find bin/ -executable -type f | xargs shellcheck + build-bazel: parameters: toolchain: @@ -59,6 +67,7 @@ workflows: pull-request: jobs: - format + - shellcheck - test-cmake: matrix: parameters: diff --git a/Dockerfile b/Dockerfile index 1e52c44e..32134faa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ env DEBIAN_FRONTEND=noninteractive run apt-get update && apt-get install --yes software-properties-common && \ add-apt-repository ppa:git-core/ppa --yes && \ apt-get update && apt-get upgrade --yes && \ - apt-get install --yes wget build-essential clang sed gdb clang-format git ssh + apt-get install --yes wget build-essential clang sed gdb clang-format git ssh shellcheck # bazelisk, a launcher for bazel. `bazelisk --help` will cause the latest # version to be downloaded. From f789c1c07fe84aeaebd7c460cbb0514dc897750f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 18:30:41 -0500 Subject: [PATCH 26/54] tricky rebase... --- bin/test | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/test b/bin/test index f7cddf38..46ca7707 100755 --- a/bin/test +++ b/bin/test @@ -41,6 +41,7 @@ cd .build cmake .. $coverage_flags -DBUILD_TESTING=1 # Clean up any code coverage artifacts. find . -type f -name '*.gcda' -print0 | xargs -0 rm -f +# shellcheck disable=SC2086 make -j "$MAKE_JOB_COUNT" $verbosity_flags if [ "$build_only" -eq 1 ]; then From 46cc7e5ef0d2c4715949fde3cc4560e19299fe75 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 14:48:39 -0500 Subject: [PATCH 27/54] add a class TraceID for future use --- BUILD.bazel | 2 + CMakeLists.txt | 2 + src/datadog/trace_id.cpp | 94 ++++++++++++++++++++++++++++++++++++++++ src/datadog/trace_id.h | 35 +++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 src/datadog/trace_id.cpp create mode 100644 src/datadog/trace_id.h diff --git a/BUILD.bazel b/BUILD.bazel index d78fd98a..82f024cf 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -47,6 +47,7 @@ cc_library( "src/datadog/threaded_event_scheduler.cpp", "src/datadog/tracer_config.cpp", "src/datadog/tracer.cpp", + "src/datadog/trace_id.cpp", "src/datadog/trace_sampler_config.cpp", "src/datadog/trace_sampler.cpp", "src/datadog/trace_segment.cpp", @@ -101,6 +102,7 @@ cc_library( "src/datadog/threaded_event_scheduler.h", "src/datadog/tracer_config.h", "src/datadog/tracer.h", + "src/datadog/trace_id.h", "src/datadog/trace_sampler_config.h", "src/datadog/trace_sampler.h", "src/datadog/trace_segment.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f2442da..9a3054e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/threaded_event_scheduler.cpp src/datadog/tracer_config.cpp src/datadog/tracer.cpp + src/datadog/trace_id.cpp src/datadog/trace_sampler_config.cpp src/datadog/trace_sampler.cpp src/datadog/trace_segment.cpp @@ -172,6 +173,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/threaded_event_scheduler.h src/datadog/tracer_config.h src/datadog/tracer.h + src/datadog/trace_id.h src/datadog/trace_sampler_config.h src/datadog/trace_sampler.h src/datadog/trace_segment.h diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp new file mode 100644 index 00000000..ba90b13f --- /dev/null +++ b/src/datadog/trace_id.cpp @@ -0,0 +1,94 @@ +#include "trace_id.h" + +#include "hex.h" +#include "parse_util.h" + +namespace datadog { +namespace tracing { + +TraceID::TraceID() : low(0) {} + +TraceID::TraceID(std::uint64_t low) : low(low) {} + +TraceID::TraceID(std::uint64_t low, std::uint64_t high) + : low(low), high(high) {} + +std::string TraceID::hex() const { + std::string result; + if (high) { + result += datadog::tracing::hex(*high); + } + result += datadog::tracing::hex(low); + return result; +} + +std::string TraceID::low_dec() const { return std::to_string(low); } + +Expected TraceID::parse_hex(StringView input) { + const auto parse_hex_piece = + [&](StringView piece) -> Expected { + auto result = parse_uint64(piece, 16); + if (auto *error = result.if_error()) { + std::string prefix = "Unable to parse trace ID from \""; + append(prefix, input); + prefix += "\": "; + return error->with_prefix(prefix); + } + return result; + }; + + // A 64-bit integer is at most 32 hex characters. If the input is no + // longer than that, then it will all fit in `TraceID::low`. + if (input.size() <= 32) { + auto result = parse_hex_piece(input); + if (auto *error = result.if_error()) { + return std::move(*error); + } + return TraceID(*result); + } + + // Parse the lower part and the higher part separately. + const auto divider = input.begin() + (input.size() - 32); + const auto high_hex = range(input.begin(), divider); + const auto low_hex = range(divider, input.end()); + TraceID trace_id; + + auto result = parse_hex_piece(low_hex); + if (auto *error = result.if_error()) { + return std::move(*error); + } + trace_id.low = *result; + + result = parse_hex_piece(high_hex); + if (auto *error = result.if_error()) { + return std::move(*error); + } + trace_id.high = *result; + + return trace_id; +} + +bool operator==(TraceID left, TraceID right) { + return left.low == right.low && left.high == right.high; +} + +bool operator!=(TraceID left, TraceID right) { + return left.low != right.low || left.high != right.high; +} + +bool operator==(TraceID left, std::uint64_t right) { + if (left.high) { + return *left.high == 0 && left.low == right; + } + return left.low == right; +} + +bool operator!=(TraceID left, std::uint64_t right) { + if (left.high) { + return *left.high != 0 || left.low != right; + } + return left.low != right; +} + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h new file mode 100644 index 00000000..d74c7ec6 --- /dev/null +++ b/src/datadog/trace_id.h @@ -0,0 +1,35 @@ +#pragma once + +// TODO + +#include +#include + +#include "expected.h" +#include "optional.h" +#include "string_view.h" + +namespace datadog { +namespace tracing { + +struct TraceID { + std::uint64_t low; + Optional high; + + TraceID(); + explicit TraceID(std::uint64_t low); + TraceID(std::uint64_t low, std::uint64_t high); + + std::string hex() const; + std::string low_dec() const; + + static Expected parse_hex(StringView); +}; + +bool operator==(TraceID, TraceID); +bool operator!=(TraceID, TraceID); +bool operator==(TraceID, std::uint64_t); +bool operator!=(TraceID, std::uint64_t); + +} // namespace tracing +} // namespace datadog From 496c0a6e4325c41624aab7ae02da1bfc4821f4c7 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 14:57:39 -0500 Subject: [PATCH 28/54] TODO: AMEND: here we go... --- src/datadog/span.cpp | 2 +- src/datadog/span.h | 3 ++- src/datadog/span_data.cpp | 2 +- src/datadog/span_data.h | 3 ++- src/datadog/trace_segment.cpp | 4 ++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/datadog/span.cpp b/src/datadog/span.cpp index 0156b86c..70ba9f88 100644 --- a/src/datadog/span.cpp +++ b/src/datadog/span.cpp @@ -62,7 +62,7 @@ void Span::inject(DictWriter& writer) const { std::uint64_t Span::id() const { return data_->span_id; } -std::uint64_t Span::trace_id() const { return data_->trace_id; } +TraceID Span::trace_id() const { return data_->trace_id; } Optional Span::parent_id() const { if (data_->parent_id == 0) { diff --git a/src/datadog/span.h b/src/datadog/span.h index 4d7512fe..316eb1f5 100644 --- a/src/datadog/span.h +++ b/src/datadog/span.h @@ -49,6 +49,7 @@ #include "id_generator.h" #include "optional.h" #include "string_view.h" +#include "trace_id.h" namespace datadog { namespace tracing { @@ -95,7 +96,7 @@ class Span { // Return this span's ID (span ID). std::uint64_t id() const; // Return the ID of the trace of which this span is a part. - std::uint64_t trace_id() const; + TraceID trace_id() const; // Return the ID of this span's parent span, or return null if this span has // no parent. Optional parent_id() const; diff --git a/src/datadog/span_data.cpp b/src/datadog/span_data.cpp index fea93cad..b168baac 100644 --- a/src/datadog/span_data.cpp +++ b/src/datadog/span_data.cpp @@ -78,7 +78,7 @@ Expected msgpack_encode(std::string& destination, const SpanData& span) { return msgpack::pack_string(destination, span.resource); }, "trace_id", [&](auto& destination) { - msgpack::pack_integer(destination, span.trace_id); + msgpack::pack_integer(destination, span.trace_id.low); return Expected{}; }, "span_id", [&](auto& destination) { diff --git a/src/datadog/span_data.h b/src/datadog/span_data.h index 3e9b00fa..9c0ab0ee 100644 --- a/src/datadog/span_data.h +++ b/src/datadog/span_data.h @@ -12,6 +12,7 @@ #include "expected.h" #include "optional.h" #include "string_view.h" +#include "trace_id.h" namespace datadog { namespace tracing { @@ -24,7 +25,7 @@ struct SpanData { std::string service_type; std::string name; std::string resource; - std::uint64_t trace_id = 0; + TraceID trace_id; std::uint64_t span_id = 0; std::uint64_t parent_id = 0; TimePoint start; diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 11040a66..11642615 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -269,7 +269,7 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { for (const auto style : injection_styles_) { switch (style) { case PropagationStyle::DATADOG: - writer.set("x-datadog-trace-id", std::to_string(span.trace_id)); + writer.set("x-datadog-trace-id", std::to_string(span.trace_id.lower)); writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", std::to_string(sampling_priority)); @@ -280,7 +280,7 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { spans_.front()->tags, *logger_); break; case PropagationStyle::B3: - writer.set("x-b3-traceid", hex(span.trace_id)); + writer.set("x-b3-traceid", span.trace_id.hex()); writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); if (origin_) { From e17d8e4d146c4705218fa23f27b30a3480668daf Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 16:53:49 -0500 Subject: [PATCH 29/54] adjusted existing unit tests --- src/datadog/expected.h | 2 - src/datadog/extracted_data.h | 9 +-- src/datadog/limiter.cpp | 1 - src/datadog/trace_id.cpp | 19 +++-- src/datadog/trace_id.h | 7 +- src/datadog/trace_sampler.cpp | 4 +- src/datadog/trace_segment.cpp | 10 ++- src/datadog/trace_segment.h | 2 - src/datadog/tracer.cpp | 35 +++++---- src/datadog/tracer_config.h | 1 - src/datadog/w3c_propagation.cpp | 36 ++++------ src/datadog/w3c_propagation.h | 6 +- test/test_span.cpp | 8 ++- test/test_tracer.cpp | 124 +++++++++++++++++++------------- 14 files changed, 143 insertions(+), 121 deletions(-) diff --git a/src/datadog/expected.h b/src/datadog/expected.h index f442fc55..71b74d17 100644 --- a/src/datadog/expected.h +++ b/src/datadog/expected.h @@ -17,7 +17,6 @@ // auto maybe_int = parse_integer("the answer"); // // using the `if_error` method // if (auto *error = maybe_int.if_error()) { -// std::cerr << "parse_integer returned error: " << *error << '\n'; // return int(error->code); // } // @@ -26,7 +25,6 @@ // maybe_int = parse_integer("one hundred twenty-three"); // // use the `error` method. // if (!maybe_int) { -// std::cerr << "parse_integer returned error: " << maybe_int.error() << // '\n'; return int(maybe_int.error().code); // } // diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index 0f182d7f..33b72215 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -9,22 +9,17 @@ #include #include "optional.h" +#include "trace_id.h" namespace datadog { namespace tracing { struct ExtractedData { - Optional trace_id; + Optional trace_id; Optional parent_id; Optional origin; std::vector> trace_tags; Optional sampling_priority; - // If this `ExtractedData` was created on account of `PropagationStyle::W3C` - // and trace context was successfully extracted, then `full_w3c_trace_id_hex` - // contains the hex-encoded 128-bit trace ID. `trace_id` will be the least - // significant 64 bits of the same value. `full_w3c_trace_id_hex` is used for - // the `W3C` injection style. - Optional full_w3c_trace_id_hex; // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, // then `additional_w3c_tracestate` contains the parts of the "tracestate" // header that are not the "dd" (Datadog) entry. If there are no other parts, diff --git a/src/datadog/limiter.cpp b/src/datadog/limiter.cpp index 63f66b38..80c0f01f 100644 --- a/src/datadog/limiter.cpp +++ b/src/datadog/limiter.cpp @@ -2,7 +2,6 @@ #include #include -#include #include namespace datadog { diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index ba90b13f..5ce91525 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -15,14 +15,19 @@ TraceID::TraceID(std::uint64_t low, std::uint64_t high) std::string TraceID::hex() const { std::string result; - if (high) { - result += datadog::tracing::hex(*high); + if (high && *high) { + result += ::datadog::tracing::hex(*high); } - result += datadog::tracing::hex(low); + result += ::datadog::tracing::hex(low); return result; } -std::string TraceID::low_dec() const { return std::to_string(low); } +std::string TraceID::debug() const { + if (high) { + return "0x" + hex(); + } + return std::to_string(low); +} Expected TraceID::parse_hex(StringView input) { const auto parse_hex_piece = @@ -37,9 +42,9 @@ Expected TraceID::parse_hex(StringView input) { return result; }; - // A 64-bit integer is at most 32 hex characters. If the input is no + // A 64-bit integer is at most 16 hex characters. If the input is no // longer than that, then it will all fit in `TraceID::low`. - if (input.size() <= 32) { + if (input.size() <= 16) { auto result = parse_hex_piece(input); if (auto *error = result.if_error()) { return std::move(*error); @@ -48,7 +53,7 @@ Expected TraceID::parse_hex(StringView input) { } // Parse the lower part and the higher part separately. - const auto divider = input.begin() + (input.size() - 32); + const auto divider = input.begin() + (input.size() - 16); const auto high_hex = range(input.begin(), divider); const auto low_hex = range(divider, input.end()); TraceID trace_id; diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h index d74c7ec6..ae9308e4 100644 --- a/src/datadog/trace_id.h +++ b/src/datadog/trace_id.h @@ -1,6 +1,6 @@ #pragma once -// TODO +// TODO: document #include #include @@ -16,13 +16,16 @@ struct TraceID { std::uint64_t low; Optional high; + // TODO: document TraceID(); explicit TraceID(std::uint64_t low); TraceID(std::uint64_t low, std::uint64_t high); + // TODO: document std::string hex() const; - std::string low_dec() const; + std::string debug() const; + // TODO: document static Expected parse_hex(StringView); }; diff --git a/src/datadog/trace_sampler.cpp b/src/datadog/trace_sampler.cpp index 1938b57a..ae74865b 100644 --- a/src/datadog/trace_sampler.cpp +++ b/src/datadog/trace_sampler.cpp @@ -40,7 +40,7 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { decision.limiter_max_per_second = limiter_max_per_second_; decision.configured_rate = rule.sample_rate; const std::uint64_t threshold = max_id_from_rate(rule.sample_rate); - if (knuth_hash(span.trace_id) < threshold) { + if (knuth_hash(span.trace_id.low) < threshold) { const auto result = limiter_.allow(); if (result.allowed) { decision.priority = int(SamplingPriority::USER_KEEP); @@ -75,7 +75,7 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { } const std::uint64_t threshold = max_id_from_rate(*decision.configured_rate); - if (knuth_hash(span.trace_id) < threshold) { + if (knuth_hash(span.trace_id.low) < threshold) { decision.priority = int(SamplingPriority::AUTO_KEEP); } else { decision.priority = int(SamplingPriority::AUTO_DROP); diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 11642615..61ee2962 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -66,7 +66,6 @@ TraceSegment::TraceSegment( std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, - Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root) @@ -82,7 +81,6 @@ TraceSegment::TraceSegment( trace_tags_(std::move(trace_tags)), num_finished_spans_(0), sampling_decision_(std::move(sampling_decision)), - full_w3c_trace_id_hex_(std::move(full_w3c_trace_id_hex)), additional_w3c_tracestate_(std::move(additional_w3c_tracestate)), additional_datadog_w3c_tracestate_( std::move(additional_datadog_w3c_tracestate)) { @@ -269,7 +267,7 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { for (const auto style : injection_styles_) { switch (style) { case PropagationStyle::DATADOG: - writer.set("x-datadog-trace-id", std::to_string(span.trace_id.lower)); + writer.set("x-datadog-trace-id", std::to_string(span.trace_id.low)); writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", std::to_string(sampling_priority)); @@ -290,9 +288,9 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { spans_.front()->tags, *logger_); break; case PropagationStyle::W3C: - writer.set("traceparent", - encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, - span.span_id, sampling_priority)); + writer.set( + "traceparent", + encode_traceparent(span.trace_id, span.span_id, sampling_priority)); writer.set("tracestate", encode_tracestate(sampling_priority, origin_, trace_tags, additional_datadog_w3c_tracestate_, diff --git a/src/datadog/trace_segment.h b/src/datadog/trace_segment.h index 0a8ccc42..d0d9004b 100644 --- a/src/datadog/trace_segment.h +++ b/src/datadog/trace_segment.h @@ -67,7 +67,6 @@ class TraceSegment { std::vector> spans_; std::size_t num_finished_spans_; Optional sampling_decision_; - Optional full_w3c_trace_id_hex_; Optional additional_w3c_tracestate_; Optional additional_datadog_w3c_tracestate_; bool awaiting_delegated_sampling_decision_ = false; @@ -83,7 +82,6 @@ class TraceSegment { Optional origin, std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, - Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root); diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index dcc5c142..201248a8 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -83,7 +83,9 @@ Expected extract_datadog( if (auto* error = trace_id.if_error()) { return std::move(*error); } - result.trace_id = *trace_id; + if (*trace_id) { + result.trace_id = TraceID(**trace_id); + } auto parent_id = extract_id_header(headers, "x-datadog-parent-id", "parent span", "Datadog", 10); @@ -125,11 +127,16 @@ Expected extract_b3( std::unordered_map& span_tags, Logger& logger) { ExtractedData result; - auto trace_id = extract_id_header(headers, "x-b3-traceid", "trace", "B3", 16); - if (auto* error = trace_id.if_error()) { - return std::move(*error); + if (auto found = headers.lookup("x-b3-traceid")) { + auto parsed = TraceID::parse_hex(*found); + if (auto* error = parsed.if_error()) { + std::string prefix = "Could not extract B3-style trace ID from \""; + append(prefix, *found); + prefix += "\": "; + return error->with_prefix(prefix); + } + result.trace_id = *parsed; } - result.trace_id = *trace_id; auto parent_id = extract_id_header(headers, "x-b3-spanid", "parent span", "B3", 16); @@ -241,8 +248,11 @@ Span Tracer::create_span() { return create_span(SpanConfig{}); } Span Tracer::create_span(const SpanConfig& config) { auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); - span_data->span_id = generator_(); - span_data->trace_id = span_data->span_id; + span_data->trace_id.low = generator_(); + if (/* TODO: feature flag */ true) { + span_data->trace_id.high = generator_(); + } + span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; const auto span_data_ptr = span_data.get(); @@ -250,8 +260,7 @@ Span Tracer::create_span(const SpanConfig& config) { logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, std::vector>{} /* trace_tags */, - nullopt /* sampling_decision */, nullopt /* full_w3c_trace_id_hex */, - nullopt /* additional_w3c_tracestate */, + nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; @@ -300,8 +309,8 @@ Expected Tracer::extract_span(const DictReader& reader, } auto& [trace_id, parent_id, origin, trace_tags, sampling_priority, - full_w3c_trace_id_hex, additional_w3c_tracestate, - additional_datadog_w3c_tracestate] = extracted_data; + additional_w3c_tracestate, additional_datadog_w3c_tracestate] = + extracted_data; // Some information might be missing. // Here are the combinations considered: @@ -332,7 +341,7 @@ Expected Tracer::extract_span(const DictReader& reader, std::string message; message += "There's no parent span ID to extract, but there is a trace ID: "; - message += std::to_string(*trace_id); + message += trace_id->debug(); return Error{Error::MISSING_PARENT_SPAN_ID, std::move(message)}; } @@ -369,7 +378,7 @@ Expected Tracer::extract_span(const DictReader& reader, logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, std::move(origin), tags_header_max_size_, std::move(trace_tags), std::move(sampling_decision), - std::move(full_w3c_trace_id_hex), std::move(additional_w3c_tracestate), + std::move(additional_w3c_tracestate), std::move(additional_datadog_w3c_tracestate), std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; diff --git a/src/datadog/tracer_config.h b/src/datadog/tracer_config.h index 2916aec0..901dd1c8 100644 --- a/src/datadog/tracer_config.h +++ b/src/datadog/tracer_config.h @@ -87,7 +87,6 @@ struct TracerConfig { // `logger` specifies how the tracer will issue diagnostic messages. If // `logger` is null, then it defaults to a logger that inserts into - // `std::cerr`. std::shared_ptr logger = nullptr; // `log_on_startup` indicates whether the tracer will log a banner of diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index a376dd26..80e76e43 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -48,11 +48,11 @@ Optional extract_traceparent(ExtractedData& result, static const auto& pattern = "([0-9a-f]{2})" // hex version number (match group 1) "-" - "([0-9a-f]{16}([0-9a-f]{16}))" // hex trace ID (match groups 2 and 3) + "([0-9a-f]{32})" // hex trace ID (match group 2) "-" - "([0-9a-f]{16})" // hex parent span ID (match group 4) + "([0-9a-f]{16})" // hex parent span ID (match group 3) "-" - "([0-9a-f]{2})" // hex "trace-flags" (match group 5) + "([0-9a-f]{2})" // hex "trace-flags" (match group 4) "(?:$|-.*)"; // either the end, or a hyphen preceding further fields static const std::regex regex{pattern}; @@ -62,7 +62,7 @@ Optional extract_traceparent(ExtractedData& result, } assert(match.ready()); - assert(match.size() == 5 + 1); + assert(match.size() == 4 + 1); const auto to_string_view = [](const auto& submatch) { assert(submatch.first <= submatch.second); @@ -74,20 +74,17 @@ Optional extract_traceparent(ExtractedData& result, return "invalid_version"; } - result.full_w3c_trace_id_hex = std::string{to_string_view(match[2])}; - if (result.full_w3c_trace_id_hex->find_first_not_of('0') == - std::string::npos) { + result.trace_id = *TraceID::parse_hex(to_string_view(match[2])); + if (result.trace_id == 0) { return "trace_id_zero"; } - result.trace_id = *parse_uint64(to_string_view(match[3]), 16); - - result.parent_id = *parse_uint64(to_string_view(match[4]), 16); + result.parent_id = *parse_uint64(to_string_view(match[3]), 16); if (*result.parent_id == 0) { return "parent_id_zero"; } - const auto flags = *parse_uint64(to_string_view(match[5]), 16); + const auto flags = *parse_uint64(to_string_view(match[4]), 16); result.sampling_priority = int(flags & 1); return nullopt; @@ -294,25 +291,20 @@ Expected extract_w3c( return result; } -std::string encode_traceparent( - std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, - std::uint64_t span_id, int sampling_priority) { +std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id, + int sampling_priority) { std::string result; // version result += "00-"; // trace ID - if (full_w3c_trace_id_hex) { - result += *full_w3c_trace_id_hex; - } else { - auto hexed = hex(trace_id); - result.append(16 + (16 - hexed.size()), '0'); // leading zeroes - result += hexed; - } + auto hexed = trace_id.hex(); + result.append(32 - hexed.size(), '0'); // leading zeroes + result += hexed; result += '-'; // span ID - auto hexed = hex(span_id); + hexed = hex(span_id); result.append(16 - hexed.size(), '0'); // leading zeroes result += hexed; result += '-'; diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 57946ca9..dd4f6f25 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -11,6 +11,7 @@ #include "expected.h" #include "extracted_data.h" #include "optional.h" +#include "trace_id.h" namespace datadog { namespace tracing { @@ -31,9 +32,8 @@ Expected extract_w3c( // `trace_id` or the optionally specified `full_w3c_trace_id_hex` as the trace // ID, the specified `span_id` as the parent ID, and trace flags deduced from // the specified `sampling_priority`. -std::string encode_traceparent( - std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, - std::uint64_t span_id, int sampling_priority); +std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id, + int sampling_priority); // Return a value for the "tracestate" header containing the specified fields. std::string encode_tracestate( diff --git a/test/test_span.cpp b/test/test_span.cpp index ada7df6f..e0dd58ed 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -392,11 +392,11 @@ TEST_CASE("injection") { const auto& headers = writer.items; REQUIRE(headers.at("x-datadog-trace-id") == - std::to_string(span.trace_id())); + std::to_string(span.trace_id().low)); REQUIRE(headers.at("x-datadog-parent-id") == std::to_string(span.id())); REQUIRE(headers.at("x-datadog-sampling-priority") == std::to_string(priority)); - REQUIRE(headers.at("x-b3-traceid") == hex(span.trace_id())); + REQUIRE(headers.at("x-b3-traceid") == span.trace_id().hex()); REQUIRE(headers.at("x-b3-spanid") == hex(span.id())); REQUIRE(headers.at("x-b3-sampled") == std::to_string(int(priority > 0))); } @@ -536,8 +536,10 @@ TEST_CASE("injecting W3C traceparent header") { const auto found = output_headers.find("traceparent"); REQUIRE(found != output_headers.end()); // The "cafebabe"s come from `expected_id`. + // It appears twice in the trace ID because the generator is invoked twice: + // once for the low 64 bits, and again for the high 64 bits. const std::string expected = - "00-000000000000000000000000cafebabe-00000000cafebabe-" + + "00-0000000000000000cafebabecafebabe-00000000cafebabe-" + expected_flags; REQUIRE(found->second == expected); } diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 0ad69d04..cebd11dd 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -261,6 +262,7 @@ TEST_CASE("span extraction") { SECTION("extraction failures") { struct TestCase { + int line; std::string name; std::vector extraction_styles; std::unordered_map headers; @@ -269,74 +271,94 @@ TEST_CASE("span extraction") { }; auto test_case = GENERATE(values({ - {"no span", {PropagationStyle::DATADOG}, {}, Error::NO_SPAN_TO_EXTRACT}, - {"missing trace ID", + {__LINE__, + "no span", + {PropagationStyle::DATADOG}, + {}, + Error::NO_SPAN_TO_EXTRACT}, + {__LINE__, + "missing trace ID", {PropagationStyle::DATADOG}, {{"x-datadog-parent-id", "456"}}, Error::MISSING_TRACE_ID}, - {"missing parent span ID", + {__LINE__, + "missing parent span ID", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}}, Error::MISSING_PARENT_SPAN_ID}, - {"missing parent span ID, but it's ok because origin", + {__LINE__, + "missing parent span ID, but it's ok because origin", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-origin", "anything"}}, nullopt}, - {"bad x-datadog-trace-id", + {__LINE__, + "bad x-datadog-trace-id", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "f"}, {"x-datadog-parent-id", "456"}}, Error::INVALID_INTEGER}, - {"bad x-datadog-trace-id (2)", + {__LINE__, + "bad x-datadog-trace-id (2)", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "99999999999999999999999999"}, {"x-datadog-parent-id", "456"}}, Error::OUT_OF_RANGE_INTEGER}, - {"bad x-datadog-parent-id", + {__LINE__, + "bad x-datadog-parent-id", {PropagationStyle::DATADOG}, {{"x-datadog-parent-id", "f"}, {"x-datadog-trace-id", "456"}}, Error::INVALID_INTEGER}, - {"bad x-datadog-parent-id (2)", + {__LINE__, + "bad x-datadog-parent-id (2)", {PropagationStyle::DATADOG}, {{"x-datadog-parent-id", "99999999999999999999999999"}, {"x-datadog-trace-id", "456"}}, Error::OUT_OF_RANGE_INTEGER}, - {"bad x-datadog-sampling-priority", + {__LINE__, + "bad x-datadog-sampling-priority", {PropagationStyle::DATADOG}, {{"x-datadog-parent-id", "123"}, {"x-datadog-trace-id", "456"}, {"x-datadog-sampling-priority", "keep"}}, Error::INVALID_INTEGER}, - {"bad x-datadog-sampling-priority (2)", + {__LINE__, + "bad x-datadog-sampling-priority (2)", {PropagationStyle::DATADOG}, {{"x-datadog-parent-id", "123"}, {"x-datadog-trace-id", "456"}, {"x-datadog-sampling-priority", "99999999999999999999999999"}}, Error::OUT_OF_RANGE_INTEGER}, - {"bad x-b3-traceid", + {__LINE__, + "bad x-b3-traceid", {PropagationStyle::B3}, {{"x-b3-traceid", "0xdeadbeef"}, {"x-b3-spanid", "def"}}, Error::INVALID_INTEGER}, - {"bad x-b3-traceid (2)", + {__LINE__, + "bad x-b3-traceid (2)", {PropagationStyle::B3}, - {{"x-b3-traceid", "ffffffffffffffffffffffffffffff"}, + {{"x-b3-traceid", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, {"x-b3-spanid", "def"}}, Error::OUT_OF_RANGE_INTEGER}, - {"bad x-b3-spanid", + {__LINE__, + "bad x-b3-spanid", {PropagationStyle::B3}, {{"x-b3-spanid", "0xdeadbeef"}, {"x-b3-traceid", "def"}}, Error::INVALID_INTEGER}, - {"bad x-b3-spanid (2)", + {__LINE__, + "bad x-b3-spanid (2)", {PropagationStyle::B3}, {{"x-b3-spanid", "ffffffffffffffffffffffffffffff"}, {"x-b3-traceid", "def"}}, Error::OUT_OF_RANGE_INTEGER}, - {"bad x-b3-sampled", + {__LINE__, + "bad x-b3-sampled", {PropagationStyle::B3}, {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}, {"x-b3-sampled", "true"}}, Error::INVALID_INTEGER}, - {"bad x-b3-sampled (2)", + {__LINE__, + "bad x-b3-sampled (2)", {PropagationStyle::B3}, {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}, @@ -344,6 +366,7 @@ TEST_CASE("span extraction") { Error::OUT_OF_RANGE_INTEGER}, })); + CAPTURE(test_case.line); CAPTURE(test_case.name); config.extraction_styles = test_case.extraction_styles; @@ -380,7 +403,7 @@ TEST_CASE("span extraction") { std::string name; std::vector extraction_styles; std::unordered_map headers; - std::uint64_t expected_trace_id; + TraceID expected_trace_id; Optional expected_parent_id; Optional expected_sampling_priority; }; @@ -391,19 +414,19 @@ TEST_CASE("span extraction") { {{"x-datadog-trace-id", "123"}, {"x-datadog-parent-id", "456"}, {"x-datadog-sampling-priority", "2"}}, - 123, + TraceID(123), 456, 2}, {"datadog style without sampling priority", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-parent-id", "456"}}, - 123, + TraceID(123), 456, nullopt}, {"datadog style without sampling priority and without parent ID", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-origin", "whatever"}}, - 123, + TraceID(123), nullopt, nullopt}, {"B3 style", @@ -411,13 +434,13 @@ TEST_CASE("span extraction") { {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}, {"x-b3-sampled", "0"}}, - 0xabc, + TraceID(0xabc), 0xdef, 0}, {"B3 style without sampling priority", {PropagationStyle::B3}, {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}}, - 0xabc, + TraceID(0xabc), 0xdef, nullopt}, {"Datadog overriding B3", @@ -428,7 +451,7 @@ TEST_CASE("span extraction") { {"x-b3-traceid", "fff"}, {"x-b3-spanid", "ef"}, {"x-b3-sampled", "0"}}, - 255, + TraceID(255), 14, 0}, {"Datadog overriding B3, without sampling priority", @@ -437,19 +460,19 @@ TEST_CASE("span extraction") { {"x-datadog-parent-id", "14"}, {"x-b3-traceid", "fff"}, {"x-b3-spanid", "ef"}}, - 255, + TraceID(255), 14, nullopt}, {"B3 after Datadog found no context", {PropagationStyle::DATADOG, PropagationStyle::B3}, {{"x-b3-traceid", "ff"}, {"x-b3-spanid", "e"}}, - 0xff, + TraceID(0xff), 0xe, nullopt}, {"Datadog after B3 found no context", {PropagationStyle::B3, PropagationStyle::DATADOG}, {{"x-b3-traceid", "fff"}, {"x-b3-spanid", "ef"}}, - 0xfff, + TraceID(0xfff), 0xef, nullopt}, })); @@ -507,6 +530,12 @@ TEST_CASE("span extraction") { REQUIRE(result.error().code == Error::NO_SPAN_TO_EXTRACT); } + SECTION("sanity") { + // In the following tabulated cases, we assume that this hex string is a + // valid trace ID. + REQUIRE(TraceID::parse_hex("4bf92f3577b34da6a3ce929d0e0e4736")); + } + SECTION("W3C traceparent extraction") { const std::unordered_map datadog_headers{ {"x-datadog-trace-id", "18"}, @@ -519,8 +548,7 @@ TEST_CASE("span extraction") { std::string name; Optional traceparent; Optional expected_error_tag_value = {}; - Optional expected_full_trace_id = {}; - Optional expected_trace_id = {}; + Optional expected_trace_id = {}; Optional expected_parent_id = {}; Optional expected_sampling_priority = {}; }; @@ -531,46 +559,42 @@ TEST_CASE("span extraction") { {__LINE__, "valid: w3.org example 1", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", // traceparent nullopt, - "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id - 11803532876627986230ULL, // expected_trace_id + *TraceID::parse_hex("4bf92f3577b34da6a3ce929d0e0e4736"), // expected_trace_id 67667974448284343ULL, // expected_parent_id 1}, // expected_sampling_priority {__LINE__, "valid: w3.org example 2", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent nullopt, - "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id - 11803532876627986230ULL, // expected_trace_id + *TraceID::parse_hex("4bf92f3577b34da6a3ce929d0e0e4736"), // expected_trace_id 67667974448284343ULL, // expected_parent_id 0}, // expected_sampling_priority {__LINE__, "valid: future version", "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent nullopt, - "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id - 11803532876627986230ULL, // expected_trace_id + *TraceID::parse_hex("4bf92f3577b34da6a3ce929d0e0e4736"), // expected_trace_id 67667974448284343ULL, // expected_parent_id 0}, // expected_sampling_priority {__LINE__, "valid: future version with extra fields", "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00-af-delta", // traceparent nullopt, - "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id - 11803532876627986230ULL, // expected_trace_id + *TraceID::parse_hex("4bf92f3577b34da6a3ce929d0e0e4736"), // expected_trace_id 67667974448284343ULL, // expected_parent_id 0}, // expected_sampling_priority - + {__LINE__, "no traceparent", nullopt}, // traceparent - + {__LINE__, "invalid: not enough fields", "06-4bf92f3577b34da6a3ce929d0e0e4736", // traceparent "malformed_traceparent"}, // expected_error_tag_value - + {__LINE__, "invalid: missing hyphen", "064bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent "malformed_traceparent"}, // expected_error_tag_value - + {__LINE__, "invalid: extra data not preceded by hyphen", "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00af-delta", // traceparent "malformed_traceparent"}, // expected_error_tag_value @@ -687,7 +711,7 @@ TEST_CASE("span extraction") { traceparent_drop, // traceparent "", // tracestate 0}, // expected_sampling_priority - + {__LINE__, "no dd entry", traceparent_drop, // traceparent "foo=hello,@thingy/thing=wah;wah;wah", // tracestate @@ -696,7 +720,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "foo=hello,@thingy/thing=wah;wah;wah", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "empty entry", traceparent_drop, // traceparent "foo=hello,,bar=thing", // tracestate @@ -705,7 +729,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "foo=hello,,bar=thing", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "malformed entry", traceparent_drop, // traceparent "foo=hello,chicken,bar=thing", // tracestate @@ -714,7 +738,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "foo=hello,chicken,bar=thing", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "stuff before dd entry", traceparent_drop, // traceparent "foo=hello,bar=baz,dd=", // tracestate @@ -723,7 +747,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "foo=hello,bar=baz", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "stuff after dd entry", traceparent_drop, // traceparent "dd=,foo=hello,bar=baz", // tracestate @@ -732,7 +756,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "foo=hello,bar=baz", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "stuff before and after dd entry", traceparent_drop, // traceparent "chicken=yes,nuggets=yes,dd=,foo=hello,bar=baz", // tracestate @@ -741,7 +765,7 @@ TEST_CASE("span extraction") { {}, // expected_trace_tags "chicken=yes,nuggets=yes,foo=hello,bar=baz", // expected_additional_w3c_tracestate nullopt}, // expected_additional_datadog_w3c_tracestate - + {__LINE__, "dd entry with empty subentries", traceparent_drop, // traceparent "dd=foo:bar;;;;;baz:bam;;;", // tracestate @@ -808,12 +832,12 @@ TEST_CASE("span extraction") { traceparent_keep, // traceparent "dd=s:-1", // tracestate 1}, // expected_sampling_priority - + {__LINE__, "invalid sampling priority (1/2)", traceparent_drop, // traceparent "dd=s:oops", // tracestate 0}, // expected_sampling_priority - + {__LINE__, "invalid sampling priority (2/2)", traceparent_keep, // traceparent "dd=s:oops", // tracestate From 3ffa4dc8142b2a85235e8da3e945a5620633ad3b Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 18:15:19 -0500 Subject: [PATCH 30/54] zero, not null --- src/datadog/trace_id.cpp | 18 ++++++------------ src/datadog/trace_id.h | 3 +-- test/test.cpp | 10 ++++++++++ test/test.h | 3 +++ test/test_tracer.cpp | 29 ++++++++++++++++++++--------- 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index 5ce91525..c1fbc839 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -6,17 +6,17 @@ namespace datadog { namespace tracing { -TraceID::TraceID() : low(0) {} +TraceID::TraceID() : low(0), high(0) {} -TraceID::TraceID(std::uint64_t low) : low(low) {} +TraceID::TraceID(std::uint64_t low) : low(low), high(0) {} TraceID::TraceID(std::uint64_t low, std::uint64_t high) : low(low), high(high) {} std::string TraceID::hex() const { std::string result; - if (high && *high) { - result += ::datadog::tracing::hex(*high); + if (high) { + result += ::datadog::tracing::hex(high); } result += ::datadog::tracing::hex(low); return result; @@ -82,17 +82,11 @@ bool operator!=(TraceID left, TraceID right) { } bool operator==(TraceID left, std::uint64_t right) { - if (left.high) { - return *left.high == 0 && left.low == right; - } - return left.low == right; + return left.high == 0 && left.low == right; } bool operator!=(TraceID left, std::uint64_t right) { - if (left.high) { - return *left.high != 0 || left.low != right; - } - return left.low != right; + return left.high != 0 || left.low != right; } } // namespace tracing diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h index ae9308e4..cbde1ea8 100644 --- a/src/datadog/trace_id.h +++ b/src/datadog/trace_id.h @@ -6,7 +6,6 @@ #include #include "expected.h" -#include "optional.h" #include "string_view.h" namespace datadog { @@ -14,7 +13,7 @@ namespace tracing { struct TraceID { std::uint64_t low; - Optional high; + std::uint64_t high; // TODO: document TraceID(); diff --git a/test/test.cpp b/test/test.cpp index 801d68a7..6359577d 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -23,3 +23,13 @@ std::ostream& operator<<(std::ostream& stream, } } // namespace std + +namespace datadog { +namespace tracing { + +std::ostream& operator<<(std::ostream& stream, TraceID trace_id) { + return stream << trace_id.debug(); +} + +} // namespace tracing +} // namespace datadog diff --git a/test/test.h b/test/test.h index 0cf17b1b..bce5a3f6 100644 --- a/test/test.h +++ b/test/test.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -42,5 +43,7 @@ std::ostream& operator<<(std::ostream& stream, return stream << expected.error(); } +std::ostream& operator<<(std::ostream&, TraceID); + } // namespace tracing } // namespace datadog diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index cebd11dd..8ac7af96 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -400,6 +400,7 @@ TEST_CASE("span extraction") { SECTION("extracted span has the expected properties") { struct TestCase { + int line; std::string name; std::vector extraction_styles; std::unordered_map headers; @@ -409,7 +410,8 @@ TEST_CASE("span extraction") { }; auto test_case = GENERATE(values({ - {"datadog style", + {__LINE__, + "datadog style", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-parent-id", "456"}, @@ -417,19 +419,22 @@ TEST_CASE("span extraction") { TraceID(123), 456, 2}, - {"datadog style without sampling priority", + {__LINE__, + "datadog style without sampling priority", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-parent-id", "456"}}, TraceID(123), 456, nullopt}, - {"datadog style without sampling priority and without parent ID", + {__LINE__, + "datadog style without sampling priority and without parent ID", {PropagationStyle::DATADOG}, {{"x-datadog-trace-id", "123"}, {"x-datadog-origin", "whatever"}}, TraceID(123), nullopt, nullopt}, - {"B3 style", + {__LINE__, + "B3 style", {PropagationStyle::B3}, {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}, @@ -437,13 +442,15 @@ TEST_CASE("span extraction") { TraceID(0xabc), 0xdef, 0}, - {"B3 style without sampling priority", + {__LINE__, + "B3 style without sampling priority", {PropagationStyle::B3}, {{"x-b3-traceid", "abc"}, {"x-b3-spanid", "def"}}, TraceID(0xabc), 0xdef, nullopt}, - {"Datadog overriding B3", + {__LINE__, + "Datadog overriding B3", {PropagationStyle::DATADOG, PropagationStyle::B3}, {{"x-datadog-trace-id", "255"}, {"x-datadog-parent-id", "14"}, @@ -454,7 +461,8 @@ TEST_CASE("span extraction") { TraceID(255), 14, 0}, - {"Datadog overriding B3, without sampling priority", + {__LINE__, + "Datadog overriding B3, without sampling priority", {PropagationStyle::DATADOG, PropagationStyle::B3}, {{"x-datadog-trace-id", "255"}, {"x-datadog-parent-id", "14"}, @@ -463,13 +471,15 @@ TEST_CASE("span extraction") { TraceID(255), 14, nullopt}, - {"B3 after Datadog found no context", + {__LINE__, + "B3 after Datadog found no context", {PropagationStyle::DATADOG, PropagationStyle::B3}, {{"x-b3-traceid", "ff"}, {"x-b3-spanid", "e"}}, TraceID(0xff), 0xe, nullopt}, - {"Datadog after B3 found no context", + {__LINE__, + "Datadog after B3 found no context", {PropagationStyle::B3, PropagationStyle::DATADOG}, {{"x-b3-traceid", "fff"}, {"x-b3-spanid", "ef"}}, TraceID(0xfff), @@ -477,6 +487,7 @@ TEST_CASE("span extraction") { nullopt}, })); + CAPTURE(test_case.line); CAPTURE(test_case.name); config.extraction_styles = test_case.extraction_styles; From efe675f5ccbf76e1b916522c3b78e375da4121e7 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 18:36:20 -0500 Subject: [PATCH 31/54] TODO: AMEND: thinking about _dd.p.tid --- src/datadog/tracer.cpp | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 201248a8..da0e6a90 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -7,6 +7,7 @@ #include "dict_reader.h" #include "environment.h" #include "extracted_data.h" +#include "hex.h" #include "json.hpp" #include "logger.h" #include "net_util.h" @@ -114,11 +115,29 @@ Expected extract_datadog( result.origin = std::string(*origin); } - auto trace_tags = headers.lookup("x-datadog-tags"); - if (trace_tags) { + if (auto trace_tags = headers.lookup("x-datadog-tags")) { handle_trace_tags(*trace_tags, result, span_tags, logger); } + // See if the trace ID has any of its high bits set. + // TODO: There are a bunch of combinations with B3 and W3C... + // tabulate them and think about it more when you're fresh. + const auto found = std::find_if( + result.trace_tags.begin(), result.trace_tags.end(), + [](const auto& entry) { return entry.first == "_dd.p.tid"; }); + if (found != result.trace_tags.end()) { + auto high = parse_uint64(found->second, 16); + if (auto* error = high.if_error()) { + // TODO: maybe don't fail + return error->with_prefix( + "Unable to parse high bits of the trace ID in Datadog style from the " + "\"_dd.p.tid\" trace tag: "); + } + if (result.trace_id) { + result.trace_id->high = *high; + } + } + return result; } @@ -248,9 +267,11 @@ Span Tracer::create_span() { return create_span(SpanConfig{}); } Span Tracer::create_span(const SpanConfig& config) { auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); + std::vector> trace_tags; span_data->trace_id.low = generator_(); if (/* TODO: feature flag */ true) { - span_data->trace_id.high = generator_(); + const auto high = span_data->trace_id.high = generator_(); + trace_tags.emplace_back("_dd.p.tid", hex(high)); } span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; @@ -259,8 +280,8 @@ Span Tracer::create_span(const SpanConfig& config) { const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, - std::vector>{} /* trace_tags */, - nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, + std::move(trace_tags), nullopt /* sampling_decision */, + nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; From 62ed713ffa4b9be33e9719811afa0d415290bc9b Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 11 Jan 2023 18:36:20 -0500 Subject: [PATCH 32/54] TODO: AMEND: thinking about _dd.p.tid --- src/datadog/trace_segment.cpp | 5 ---- src/datadog/tracer.cpp | 50 +++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 61ee2962..dd68923d 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -281,11 +281,6 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-b3-traceid", span.trace_id.hex()); writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); - if (origin_) { - writer.set("x-datadog-origin", *origin_); - } - inject_trace_tags(writer, trace_tags, tags_header_max_size_, - spans_.front()->tags, *logger_); break; case PropagationStyle::W3C: writer.set( diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 201248a8..bdc5153a 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -7,6 +7,7 @@ #include "dict_reader.h" #include "environment.h" #include "extracted_data.h" +#include "hex.h" #include "json.hpp" #include "logger.h" #include "net_util.h" @@ -41,8 +42,27 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, } for (auto& [key, value] : *maybe_trace_tags) { - if (starts_with(key, "_dd.p.")) { - result.trace_tags.emplace_back(std::move(key), std::move(value)); + if (!starts_with(key, "_dd.p.")) { + continue; + } + + result.trace_tags.emplace_back(std::move(key), std::move(value)); + + if (key == "_dd.p.tid") { + // _dd.p.tid contains the high 64 bits of the trace ID. + auto high = parse_uint64(value, 16); + if (auto* error = high.if_error()) { + logger.log_error( + error->with_prefix("Unable to parse high bits of the trace ID in " + "Datadog style from the " + "\"_dd.p.tid\" trace tag: ")); + span_tags[tags::internal::propagation_error] = "decoding_error"; + } + // Note that this assumes the lower 64 bits of the trace ID have already + // been extracted (i.e. we look for X-Datadog-Trace-ID first). + if (result.trace_id) { + result.trace_id->high = *high; + } } } } @@ -114,8 +134,7 @@ Expected extract_datadog( result.origin = std::string(*origin); } - auto trace_tags = headers.lookup("x-datadog-tags"); - if (trace_tags) { + if (auto trace_tags = headers.lookup("x-datadog-tags")) { handle_trace_tags(*trace_tags, result, span_tags, logger); } @@ -123,8 +142,8 @@ Expected extract_datadog( } Expected extract_b3( - const DictReader& headers, - std::unordered_map& span_tags, Logger& logger) { + const DictReader& headers, std::unordered_map&, + Logger&) { ExtractedData result; if (auto found = headers.lookup("x-b3-traceid")) { @@ -160,17 +179,6 @@ Expected extract_b3( result.sampling_priority = *sampling_priority; } - // Origin and trace tags are still extracted, but from the Datadog headers. - auto origin = headers.lookup("x-datadog-origin"); - if (origin) { - result.origin = std::string(*origin); - } - - auto trace_tags = headers.lookup("x-datadog-tags"); - if (trace_tags) { - handle_trace_tags(*trace_tags, result, span_tags, logger); - } - return result; } @@ -248,9 +256,11 @@ Span Tracer::create_span() { return create_span(SpanConfig{}); } Span Tracer::create_span(const SpanConfig& config) { auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); + std::vector> trace_tags; span_data->trace_id.low = generator_(); if (/* TODO: feature flag */ true) { - span_data->trace_id.high = generator_(); + const auto high = span_data->trace_id.high = generator_(); + trace_tags.emplace_back("_dd.p.tid", hex(high)); } span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; @@ -259,8 +269,8 @@ Span Tracer::create_span(const SpanConfig& config) { const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, - std::vector>{} /* trace_tags */, - nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, + std::move(trace_tags), nullopt /* sampling_decision */, + nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; From 4b177c1bf6dc09cd98bfa13e2d5fc93bf5222dbe Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 12 Jan 2023 18:09:55 -0500 Subject: [PATCH 33/54] getting there... --- src/datadog/hex.h | 36 +++++++++++++---- src/datadog/id_generator.cpp | 31 +++++++++++--- src/datadog/id_generator.h | 18 +++++---- src/datadog/span.cpp | 3 +- src/datadog/span.h | 17 ++++---- src/datadog/trace_id.cpp | 11 +++-- src/datadog/trace_id.h | 2 +- src/datadog/trace_segment.cpp | 8 +++- src/datadog/tracer.cpp | 30 +++++++++----- src/datadog/tracer.h | 8 +++- src/datadog/tracer_config.cpp | 3 ++ src/datadog/tracer_config.h | 11 +++++ src/datadog/w3c_propagation.cpp | 12 +++--- test/test_span.cpp | 71 ++++++++++++++++----------------- test/test_trace_sampler.cpp | 2 +- test/test_tracer_config.cpp | 15 ++++--- 16 files changed, 179 insertions(+), 99 deletions(-) diff --git a/src/datadog/hex.h b/src/datadog/hex.h index dec38a2e..593ddbbc 100644 --- a/src/datadog/hex.h +++ b/src/datadog/hex.h @@ -1,6 +1,6 @@ #pragma once -// This component provides a function, `hex`, for formatting an integral value +// This component provides functions for formatting an unsigned integral value // in hexadecimal. #include @@ -12,12 +12,14 @@ namespace datadog { namespace tracing { -// Return the specified `value` formatted as a lower-case hexadecimal string -// without any leading zeroes. -template -std::string hex(Integer value) { - // 4 bits per hex digit char, and then +1 char for possible minus sign - char buffer[std::numeric_limits::digits / 4 + 1]; +// Return the specified unsigned `value` formatted as a lower-case hexadecimal +// string without any leading zeroes. +template +std::string hex(UnsignedInteger value) { + static_assert(!std::numeric_limits::is_signed); + + // 4 bits per hex digit char + char buffer[std::numeric_limits::digits / 4]; const int base = 16; auto result = @@ -27,5 +29,25 @@ std::string hex(Integer value) { return std::string{std::begin(buffer), result.ptr}; } +// Return the specified unsigned `value` formatted as a lower-case hexadecimal +// string with leading zeroes. +template +std::string hex_padded(UnsignedInteger value) { + static_assert(!std::numeric_limits::is_signed); + + // 4 bits per hex digit char. + char buffer[std::numeric_limits::digits / 4]; + + const int base = 16; + auto result = + std::to_chars(std::begin(buffer), std::end(buffer), value, base); + assert(result.ec == std::errc()); + + const auto num_zeroes = sizeof(buffer) - (result.ptr - std::begin(buffer)); + std::string padded(num_zeroes, '0'); + padded.append(std::begin(buffer), result.ptr); + return padded; +} + } // namespace tracing } // namespace datadog \ No newline at end of file diff --git a/src/datadog/id_generator.cpp b/src/datadog/id_generator.cpp index bccae8ca..5f186bef 100644 --- a/src/datadog/id_generator.cpp +++ b/src/datadog/id_generator.cpp @@ -16,7 +16,7 @@ namespace { extern "C" void on_fork(); #endif -class DefaultIDGenerator { +class Uint64Generator { std::mt19937_64 generator_; // The distribution used is for a _signed_ integer, and the default minimum // value is zero. @@ -26,7 +26,7 @@ class DefaultIDGenerator { std::uniform_int_distribution distribution_; public: - DefaultIDGenerator() { + Uint64Generator() { seed_with_random(); // If a process links to this library and then calls `fork`, the `generator_` in // the parent and child processes will produce the exact same sequence of @@ -48,17 +48,36 @@ class DefaultIDGenerator { void seed_with_random() { generator_.seed(std::random_device{}()); } }; -thread_local DefaultIDGenerator thread_local_generator; +thread_local Uint64Generator thread_local_generator; #ifndef _MSC_VER void on_fork() { thread_local_generator.seed_with_random(); } #endif -} // namespace +class DefaultIDGenerator : public IDGenerator { + const bool trace_id_128_bit_; + + public: + explicit DefaultIDGenerator(bool trace_id_128_bit) + : trace_id_128_bit_(trace_id_128_bit) {} + + TraceID trace_id() const override { + TraceID result; + result.low = thread_local_generator(); + if (trace_id_128_bit_) { + result.high = thread_local_generator(); + } + return result; + } -const IDGenerator default_id_generator = []() { - return thread_local_generator(); + std::uint64_t span_id() const override { return thread_local_generator(); } }; +} // namespace + +std::shared_ptr default_id_generator(bool trace_id_128_bit) { + return std::make_shared(trace_id_128_bit); +} + } // namespace tracing } // namespace datadog diff --git a/src/datadog/id_generator.h b/src/datadog/id_generator.h index 39363c30..a1b0bb52 100644 --- a/src/datadog/id_generator.h +++ b/src/datadog/id_generator.h @@ -3,23 +3,27 @@ // This component provides facilities for generating sequences of IDs used as // span IDs and trace IDs. // -// `IDGenerator` is an alias for `std::function`. +// `IDGenerator` is an interface for generating trace IDs and span IDs. // // `default_id_generator` is an `IDGenerator` that produces a thread-local // pseudo-random sequence of uniformly distributed 63-bit unsigned integers. The -// sequence is randomly seeded one per thread and anytime the process forks. -// The IDs are 63-bit (instead of 64-bit) to ease compatibility with peer -// runtimes that don't have a native 64-bit unsigned numeric type. +// sequence is randomly seeded once per thread and anytime the process forks. #include -#include +#include + +#include "trace_id.h" namespace datadog { namespace tracing { -using IDGenerator = std::function; +class IDGenerator { + public: + virtual std::uint64_t span_id() const = 0; + virtual TraceID trace_id() const = 0; +}; -extern const IDGenerator default_id_generator; +std::shared_ptr default_id_generator(bool trace_id_128_bit); } // namespace tracing } // namespace datadog diff --git a/src/datadog/span.cpp b/src/datadog/span.cpp index 70ba9f88..fc5e7089 100644 --- a/src/datadog/span.cpp +++ b/src/datadog/span.cpp @@ -15,7 +15,8 @@ namespace datadog { namespace tracing { Span::Span(SpanData* data, const std::shared_ptr& trace_segment, - const IDGenerator& generate_span_id, const Clock& clock) + const std::function& generate_span_id, + const Clock& clock) : trace_segment_(trace_segment), data_(data), generate_span_id_(generate_span_id), diff --git a/src/datadog/span.h b/src/datadog/span.h index 316eb1f5..534ed706 100644 --- a/src/datadog/span.h +++ b/src/datadog/span.h @@ -41,12 +41,12 @@ // via the `set_end_time` member function prior to the span's destruction. #include +#include #include #include #include "clock.h" #include "error.h" -#include "id_generator.h" #include "optional.h" #include "string_view.h" #include "trace_id.h" @@ -62,19 +62,18 @@ class TraceSegment; class Span { std::shared_ptr trace_segment_; SpanData* data_; - IDGenerator generate_span_id_; + std::function generate_span_id_; Clock clock_; Optional end_time_; public: - // Create a span whose properties are stored in the specified `data` and that - // is associated with the specified `trace_segment`. Optionally specify - // `generate_span_id` to generate IDs of child spans, and a `clock` to - // determine start and end times. If `generate_span_id` and `clock` are not - // specified`, then `default_id_generator` and `default_clock` are used - // instead respectively. + // Create a span whose properties are stored in the specified `data`, that is + // associated with the specified `trace_segment`, that uses the specified + // `generate_span_id` to generate IDs of child spans, and that uses the + // specified `clock` to determine start and end times. Span(SpanData* data, const std::shared_ptr& trace_segment, - const IDGenerator& generate_span_id, const Clock& clock); + const std::function& generate_span_id, + const Clock& clock); Span(const Span&) = delete; Span(Span&&) = default; Span& operator=(Span&&) = default; diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index c1fbc839..10c38aa0 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -13,18 +13,21 @@ TraceID::TraceID(std::uint64_t low) : low(low), high(0) {} TraceID::TraceID(std::uint64_t low, std::uint64_t high) : low(low), high(high) {} -std::string TraceID::hex() const { +std::string TraceID::hex_padded() const { + // TODO std::string result; if (high) { - result += ::datadog::tracing::hex(high); + result += ::datadog::tracing::hex_padded(high); + } else { + result.append(16, '0'); } - result += ::datadog::tracing::hex(low); + result += ::datadog::tracing::hex_padded(low); return result; } std::string TraceID::debug() const { if (high) { - return "0x" + hex(); + return "0x" + hex_padded(); } return std::to_string(low); } diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h index cbde1ea8..bcb21178 100644 --- a/src/datadog/trace_id.h +++ b/src/datadog/trace_id.h @@ -21,7 +21,7 @@ struct TraceID { TraceID(std::uint64_t low, std::uint64_t high); // TODO: document - std::string hex() const; + std::string hex_padded() const; std::string debug() const; // TODO: document diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index dd68923d..3c0c39ba 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -278,8 +278,12 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { spans_.front()->tags, *logger_); break; case PropagationStyle::B3: - writer.set("x-b3-traceid", span.trace_id.hex()); - writer.set("x-b3-spanid", hex(span.span_id)); + if (span.trace_id.high) { + writer.set("x-b3-traceid", span.trace_id.hex_padded()); + } else { + writer.set("x-b3-traceid", hex_padded(span.trace_id.low)); + } + writer.set("x-b3-spanid", hex_padded(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); break; case PropagationStyle::W3C: diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index bdc5153a..e4feb7ac 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -217,10 +217,19 @@ void log_startup_message(Logger& logger, StringView tracer_version_string, } // namespace Tracer::Tracer(const FinalizedTracerConfig& config) - : Tracer(config, default_id_generator, default_clock) {} + : Tracer(config, default_id_generator(config.trace_id_128_bit), + default_clock) {} Tracer::Tracer(const FinalizedTracerConfig& config, - const IDGenerator& generator, const Clock& clock) + const std::shared_ptr& generator) + : Tracer(config, generator, default_clock) {} + +Tracer::Tracer(const FinalizedTracerConfig& config, const Clock& clock) + : Tracer(config, default_id_generator(config.trace_id_128_bit), clock) {} + +Tracer::Tracer(const FinalizedTracerConfig& config, + const std::shared_ptr& generator, + const Clock& clock) : logger_(config.logger), collector_(/* see constructor body */), trace_sampler_( @@ -257,10 +266,9 @@ Span Tracer::create_span(const SpanConfig& config) { auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); std::vector> trace_tags; - span_data->trace_id.low = generator_(); - if (/* TODO: feature flag */ true) { - const auto high = span_data->trace_id.high = generator_(); - trace_tags.emplace_back("_dd.p.tid", hex(high)); + span_data->trace_id = generator_->trace_id(); + if (span_data->trace_id.high) { + trace_tags.emplace_back("_dd.p.tid", hex(span_data->trace_id.high)); } span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; @@ -272,7 +280,9 @@ Span Tracer::create_span(const SpanConfig& config) { std::move(trace_tags), nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); - Span span{span_data_ptr, segment, generator_, clock_}; + Span span{span_data_ptr, segment, + [generator = generator_]() { return generator->span_id(); }, + clock_}; return span; } @@ -368,7 +378,7 @@ Expected Tracer::extract_span(const DictReader& reader, assert(trace_id); span_data->apply_config(*defaults_, config, clock_); - span_data->span_id = generator_(); + span_data->span_id = generator_->span_id(); span_data->trace_id = *trace_id; span_data->parent_id = *parent_id; @@ -390,7 +400,9 @@ Expected Tracer::extract_span(const DictReader& reader, std::move(trace_tags), std::move(sampling_decision), std::move(additional_w3c_tracestate), std::move(additional_datadog_w3c_tracestate), std::move(span_data)); - Span span{span_data_ptr, segment, generator_, clock_}; + Span span{span_data_ptr, segment, + [generator = generator_]() { return generator->span_id(); }, + clock_}; return span; } diff --git a/src/datadog/tracer.h b/src/datadog/tracer.h index b894c791..b7ec2fa4 100644 --- a/src/datadog/tracer.h +++ b/src/datadog/tracer.h @@ -31,7 +31,7 @@ class Tracer { std::shared_ptr collector_; std::shared_ptr trace_sampler_; std::shared_ptr span_sampler_; - IDGenerator generator_; + std::shared_ptr generator_; Clock clock_; std::shared_ptr defaults_; std::vector injection_styles_; @@ -44,7 +44,11 @@ class Tracer { // - using the specified `generator` to create trace IDs and span IDs // - using the specified `clock` to get the current time. explicit Tracer(const FinalizedTracerConfig& config); - Tracer(const FinalizedTracerConfig& config, const IDGenerator& generator, + Tracer(const FinalizedTracerConfig& config, + const std::shared_ptr& generator); + Tracer(const FinalizedTracerConfig& config, const Clock& clock); + Tracer(const FinalizedTracerConfig& config, + const std::shared_ptr& generator, const Clock& clock); // Create a new trace and return the root span of the trace. Optionally diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 6bf95b1a..43c5037d 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -360,6 +360,9 @@ Expected finalize_config(const TracerConfig &config) { result.report_hostname = config.report_hostname; result.tags_header_size = config.tags_header_size; + // TODO: DD_TRACE_128_BIT... + result.trace_id_128_bit = config.trace_id_128_bit; + return result; } diff --git a/src/datadog/tracer_config.h b/src/datadog/tracer_config.h index 901dd1c8..1c3d69e8 100644 --- a/src/datadog/tracer_config.h +++ b/src/datadog/tracer_config.h @@ -94,6 +94,16 @@ struct TracerConfig { // `log_on_startup` is overridden by the `DD_TRACE_STARTUP_LOGS` environment // variable. bool log_on_startup = true; + + // `trace_id_128_bit` indicates whether the tracer will generate and interpret + // the higher 64 bits of trace IDs. If true, the tracer will generate 128-bit + // trace IDs, extract them from incoming trace context, inject them into + // outgoing trace context, and send them to the collector. If false, then the + // tracer will generate 64-bit trace IDs, and extract only the lower 64 bits + // of trace IDs from incoming trace context. + // `trace_id_128_bit` is overridden by the `DD_TRACE_128_BIT_TRACEID_ENABLED` + // environment variable. + bool trace_id_128_bit = false; }; // `FinalizedTracerConfig` contains `Tracer` implementation details derived from @@ -121,6 +131,7 @@ class FinalizedTracerConfig { std::size_t tags_header_size; std::shared_ptr logger; bool log_on_startup; + bool trace_id_128_bit; }; // Return a `FinalizedTracerConfig` from the specified `config` and from any diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 80e76e43..a6c6132e 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -298,15 +298,11 @@ std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id, result += "00-"; // trace ID - auto hexed = trace_id.hex(); - result.append(32 - hexed.size(), '0'); // leading zeroes - result += hexed; + result += trace_id.hex_padded(); result += '-'; // span ID - hexed = hex(span_id); - result.append(16 - hexed.size(), '0'); // leading zeroes - result += hexed; + result += hex_padded(span_id); result += '-'; // flags @@ -331,7 +327,9 @@ std::string encode_datadog_tracestate( for (const auto& [key, value] : trace_tags) { const StringView prefix = "_dd.p."; - if (!starts_with(key, prefix)) { + if (!starts_with(key, prefix) || key == "_dd.p.tid") { + // Either it's not a propagation tag, or it's one of the propagation tags + // that need not be included in tracestate. continue; } diff --git a/test/test_span.cpp b/test/test_span.cpp index e0dd58ed..5cbcf76a 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -3,6 +3,7 @@ // for propagation. #include +#include #include #include #include @@ -11,11 +12,9 @@ #include #include -#include #include #include #include -#include #include #include "matchers.h" @@ -352,23 +351,6 @@ TEST_CASE("property setters") { } } -namespace { - -template -std::string hex(Integer value) { - // 4 bits per hex digit char, and then +1 char for possible minus sign - char buffer[std::numeric_limits::digits / 4 + 1]; - - const int base = 16; - auto result = - std::to_chars(std::begin(buffer), std::end(buffer), value, base); - assert(result.ec == std::errc()); - - return std::string{std::begin(buffer), result.ptr}; -} - -} // namespace - // Trace context injection is implemented in `TraceSegment`, but it's part of // the interface of `Span`, so the test is here. TEST_CASE("injection") { @@ -380,25 +362,32 @@ TEST_CASE("injection") { auto finalized_config = finalize_config(config); REQUIRE(finalized_config); - auto generator = []() { return 42; }; - Tracer tracer{*finalized_config, generator, default_clock}; + // Override the tracer's ID generator to always return a fixed value. + struct Generator : public IDGenerator { + const std::uint64_t id; + explicit Generator(std::uint64_t id) : id(id) {} + TraceID trace_id() const override { return TraceID(id); } + std::uint64_t span_id() const override { return id; } + }; + Tracer tracer{*finalized_config, std::make_shared(42)}; SECTION("trace ID, parent ID ,and sampling priority") { auto span = tracer.create_span(); + REQUIRE(span.trace_id() == 42); + REQUIRE(span.id() == 42); + const int priority = 3; // 😱 span.trace_segment().override_sampling_priority(priority); MockDictWriter writer; span.inject(writer); const auto& headers = writer.items; - REQUIRE(headers.at("x-datadog-trace-id") == - std::to_string(span.trace_id().low)); - REQUIRE(headers.at("x-datadog-parent-id") == std::to_string(span.id())); - REQUIRE(headers.at("x-datadog-sampling-priority") == - std::to_string(priority)); - REQUIRE(headers.at("x-b3-traceid") == span.trace_id().hex()); - REQUIRE(headers.at("x-b3-spanid") == hex(span.id())); - REQUIRE(headers.at("x-b3-sampled") == std::to_string(int(priority > 0))); + REQUIRE(headers.at("x-datadog-trace-id") == "42"); + REQUIRE(headers.at("x-datadog-parent-id") == "42"); + REQUIRE(headers.at("x-datadog-sampling-priority") == "3"); + REQUIRE(headers.at("x-b3-traceid") == "000000000000002a"); + REQUIRE(headers.at("x-b3-spanid") == "000000000000002a"); + REQUIRE(headers.at("x-b3-sampled") == "1"); } SECTION("origin and trace tags") { @@ -479,8 +468,14 @@ TEST_CASE("injecting W3C traceparent header") { // Override the tracer's ID generator to always return `expected_parent_id`. constexpr std::uint64_t expected_parent_id = 0xcafebabe; - Tracer tracer{*finalized_config, [=]() { return expected_parent_id; }, - default_clock}; + struct Generator : public IDGenerator { + const std::uint64_t id; + explicit Generator(std::uint64_t id) : id(id) {} + TraceID trace_id() const override { return TraceID(id); } + std::uint64_t span_id() const override { return id; } + }; + Tracer tracer{*finalized_config, + std::make_shared(expected_parent_id)}; const std::unordered_map input_headers{ // https://www.w3.org/TR/trace-context/#examples-of-http-traceparent-headers @@ -509,10 +504,14 @@ TEST_CASE("injecting W3C traceparent header") { REQUIRE(finalized_config); // Override the tracer's ID generator to always return a fixed value. - // This will be both the trace ID and the parent (root) span ID. constexpr std::uint64_t expected_id = 0xcafebabe; - Tracer tracer{*finalized_config, [=]() { return expected_id; }, - default_clock}; + struct Generator : public IDGenerator { + const std::uint64_t id; + explicit Generator(std::uint64_t id) : id(id) {} + TraceID trace_id() const override { return TraceID(id); } + std::uint64_t span_id() const override { return id; } + }; + Tracer tracer{*finalized_config, std::make_shared(expected_id)}; auto span = tracer.create_span(); @@ -536,10 +535,8 @@ TEST_CASE("injecting W3C traceparent header") { const auto found = output_headers.find("traceparent"); REQUIRE(found != output_headers.end()); // The "cafebabe"s come from `expected_id`. - // It appears twice in the trace ID because the generator is invoked twice: - // once for the low 64 bits, and again for the high 64 bits. const std::string expected = - "00-0000000000000000cafebabecafebabe-00000000cafebabe-" + + "00-000000000000000000000000cafebabe-00000000cafebabe-" + expected_flags; REQUIRE(found->second == expected); } diff --git a/test/test_trace_sampler.cpp b/test/test_trace_sampler.cpp index f99dc749..76fb469a 100644 --- a/test/test_trace_sampler.cpp +++ b/test/test_trace_sampler.cpp @@ -133,7 +133,7 @@ TEST_CASE("trace sampling rate limiter") { TimePoint current_time = default_clock(); // Modify `current_time` to advance the clock. auto clock = [¤t_time]() { return current_time; }; - Tracer tracer{*finalized, default_id_generator, clock}; + Tracer tracer{*finalized, clock}; for (std::size_t i = 0; i < test_case.burst_size; ++i) { auto span = tracer.create_span(); diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index ab583228..e8769619 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -96,13 +96,16 @@ class SomewhatSecureTemporaryFile : public std::fstream { SomewhatSecureTemporaryFile() try { namespace fs = std::filesystem; + const auto generator = default_id_generator(false); + const auto random = [&]() { return generator->span_id(); }; + // The goal is to create a file whose name is like // "/tmp/342394898324/239489029034", where the directory under /tmp has // permissions such that only the current user can read/write/cd it. const auto tmp = fs::temp_directory_path(); const int max_attempts = 5; for (int i = 0; i < max_attempts; ++i) { - const auto dir = tmp / std::to_string(default_id_generator()); + const auto dir = tmp / std::to_string(random()); std::error_code err; if (!fs::create_directory(dir, err)) { continue; @@ -111,7 +114,7 @@ class SomewhatSecureTemporaryFile : public std::fstream { if (err) { continue; } - const auto file = dir / std::to_string(default_id_generator()); + const auto file = dir / std::to_string(random()); if (fs::exists(file, err) || err) { continue; } @@ -1164,13 +1167,13 @@ TEST_CASE("TracerConfig propagation styles") { // ts tse se tsi si // --- --- --- --- --- /* ts */{ x, true, true, true, true }, - + /* tse */{ x, x, true, false, false }, - + /* se */{ x, x, x, false, false }, - + /* tsi */{ x, x, x, x, true }, - + /* si */{ x, x, x, x, x }, }; // clang-format on From 4cadd03d42528819964f1c0c861819e118b6c3de Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 12 Jan 2023 18:17:53 -0500 Subject: [PATCH 34/54] virtual destructor in new interface --- src/datadog/id_generator.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/datadog/id_generator.h b/src/datadog/id_generator.h index a1b0bb52..32ffa2ec 100644 --- a/src/datadog/id_generator.h +++ b/src/datadog/id_generator.h @@ -19,6 +19,8 @@ namespace tracing { class IDGenerator { public: + virtual ~IDGenerator() = default; + virtual std::uint64_t span_id() const = 0; virtual TraceID trace_id() const = 0; }; From bdd7548e8dab044fd2cac5904a8c60f293e16f35 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Fri, 13 Jan 2023 11:18:26 -0500 Subject: [PATCH 35/54] revert comments removed when I /std::cerr/d earlier --- src/datadog/expected.h | 2 ++ src/datadog/tracer_config.h | 1 + 2 files changed, 3 insertions(+) diff --git a/src/datadog/expected.h b/src/datadog/expected.h index 71b74d17..f442fc55 100644 --- a/src/datadog/expected.h +++ b/src/datadog/expected.h @@ -17,6 +17,7 @@ // auto maybe_int = parse_integer("the answer"); // // using the `if_error` method // if (auto *error = maybe_int.if_error()) { +// std::cerr << "parse_integer returned error: " << *error << '\n'; // return int(error->code); // } // @@ -25,6 +26,7 @@ // maybe_int = parse_integer("one hundred twenty-three"); // // use the `error` method. // if (!maybe_int) { +// std::cerr << "parse_integer returned error: " << maybe_int.error() << // '\n'; return int(maybe_int.error().code); // } // diff --git a/src/datadog/tracer_config.h b/src/datadog/tracer_config.h index 1c3d69e8..38d4f28e 100644 --- a/src/datadog/tracer_config.h +++ b/src/datadog/tracer_config.h @@ -87,6 +87,7 @@ struct TracerConfig { // `logger` specifies how the tracer will issue diagnostic messages. If // `logger` is null, then it defaults to a logger that inserts into + // `std::cerr`. std::shared_ptr logger = nullptr; // `log_on_startup` indicates whether the tracer will log a banner of From fc9afafedc71eb19a01271a67400d24985195c19 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Fri, 13 Jan 2023 11:56:39 -0500 Subject: [PATCH 36/54] tentative: DD_TRACE_ID_128_BIT_ENABLED --- src/datadog/environment.h | 3 ++- src/datadog/tracer_config.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/datadog/environment.h b/src/datadog/environment.h index fbcd78c3..20f65f7c 100644 --- a/src/datadog/environment.h +++ b/src/datadog/environment.h @@ -46,7 +46,8 @@ namespace environment { MACRO(DD_TRACE_SAMPLING_RULES) \ MACRO(DD_TRACE_STARTUP_LOGS) \ MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ - MACRO(DD_VERSION) + MACRO(DD_VERSION) \ + MACRO(DD_TRACE_ID_128_BIT_ENABLED) #define WITH_COMMA(ARG) ARG, diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 43c5037d..3011ce02 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -360,8 +360,11 @@ Expected finalize_config(const TracerConfig &config) { result.report_hostname = config.report_hostname; result.tags_header_size = config.tags_header_size; - // TODO: DD_TRACE_128_BIT... - result.trace_id_128_bit = config.trace_id_128_bit; + if (auto enabled_env = lookup(environment::DD_TRACE_ID_128_BIT_ENABLED)) { + result.trace_id_128_bit = !falsy(*enabled_env); + } else { + result.trace_id_128_bit = config.trace_id_128_bit; + } return result; } From 3881b274d48fda6964f166b2a286e1c4b0de7a54 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 07:50:12 -0500 Subject: [PATCH 37/54] add shellcheck to bin/check --- .circleci/config.yml | 2 +- bin/check | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 82dd2a21..b06f7953 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: resource_class: small steps: - checkout - - run: find bin/ -executable -type f | xargs shellcheck + - run: find bin/ -executable -type f -print0 | xargs -0 shellcheck build-bazel: parameters: diff --git a/bin/check b/bin/check index 2d5b4fde..79c4f5b2 100755 --- a/bin/check +++ b/bin/check @@ -11,3 +11,4 @@ cd "$(dirname "$0")"/.. bin/format --dry-run -Werror bin/test bin/bazel-build +find bin/ -executable -type f -print0 | xargs -0 shellcheck From dd6509cbe21c890a589f8a3ec048135889168d84 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 07:50:40 -0500 Subject: [PATCH 38/54] test the generation of 128-bit trace IDs --- test/test_tracer.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 106f8d7f..182dfb74 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -936,3 +937,36 @@ TEST_CASE("report hostname") { REQUIRE(tracer.create_span().trace_segment().hostname() == get_hostname()); } } + +TEST_CASE("128-bit trace IDs") { + TracerConfig config; + config.defaults.service = "testsvc"; + config.trace_id_128_bit = true; + const auto collector = std::make_shared(); + config.collector = collector; + config.logger = std::make_shared(); + const auto finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized}; + + SECTION("are generated") { + const auto span = tracer.create_span(); + // The chance that it's zero is ~2**(-64), which I'm willing to neglect. + REQUIRE(span.trace_id().high != 0); + } + + SECTION("result in _dd.p.tid trace tag being sent to collector") { + TraceID generated_id; + { + const auto span = tracer.create_span(); + generated_id = span.trace_id(); + } + REQUIRE(collector->span_count() == 1); + const auto& span = collector->first_span(); + const auto found = span.tags.find("_dd.p.tid"); + REQUIRE(found != span.tags.end()); + const auto high = parse_uint64(found->second, 16); + REQUIRE(high); + REQUIRE(*high == generated_id.high); + } +} From b3749e11084f0341bdd8798ea86be4fbf462aff2 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 09:58:06 -0500 Subject: [PATCH 39/54] test the extraction of 128-bit trace IDs --- src/datadog/tracer.cpp | 4 ++-- test/test_tracer.cpp | 48 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index ab1da93b..d690feb3 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -46,8 +46,6 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, continue; } - result.trace_tags.emplace_back(std::move(key), std::move(value)); - if (key == "_dd.p.tid") { // _dd.p.tid contains the high 64 bits of the trace ID. auto high = parse_uint64(value, 16); @@ -64,6 +62,8 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, result.trace_id->high = *high; } } + + result.trace_tags.emplace_back(std::move(key), std::move(value)); } } diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 182dfb74..d3a22ff9 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -2,6 +2,7 @@ // spans and for extracting spans from propagated trace context. #include +#include #include #include #include @@ -944,7 +945,12 @@ TEST_CASE("128-bit trace IDs") { config.trace_id_128_bit = true; const auto collector = std::make_shared(); config.collector = collector; - config.logger = std::make_shared(); + const auto logger = std::make_shared(); + config.logger = logger; + config.extraction_styles.clear(); + config.extraction_styles.push_back(PropagationStyle::W3C); + config.extraction_styles.push_back(PropagationStyle::DATADOG); + config.extraction_styles.push_back(PropagationStyle::B3); const auto finalized = finalize_config(config); REQUIRE(finalized); Tracer tracer{*finalized}; @@ -961,6 +967,8 @@ TEST_CASE("128-bit trace IDs") { const auto span = tracer.create_span(); generated_id = span.trace_id(); } + CAPTURE(logger->entries); + REQUIRE(logger->error_count() == 0); REQUIRE(collector->span_count() == 1); const auto& span = collector->first_span(); const auto found = span.tags.find("_dd.p.tid"); @@ -969,4 +977,42 @@ TEST_CASE("128-bit trace IDs") { REQUIRE(high); REQUIRE(*high == generated_id.high); } + + SECTION("extracted from W3C") { + std::unordered_map headers; + headers["traceparent"] = + "00-deadbeefdeadbeefcafebabecafebabe-0000000000000001-01"; + MockDictReader reader{headers}; + const auto span = tracer.extract_span(reader); + CAPTURE(logger->entries); + REQUIRE(logger->error_count() == 0); + REQUIRE(span); + REQUIRE(hex(span->trace_id().high) == "deadbeefdeadbeef"); + } + + SECTION("extracted from Datadog (_dd.p.tid)") { + std::unordered_map headers; + headers["x-datadog-trace-id"] = "4"; + headers["x-datadog-parent-id"] = "42"; + headers["x-datadog-tags"] = "_dd.p.tid=beef"; + MockDictReader reader{headers}; + const auto span = tracer.extract_span(reader); + CAPTURE(logger->entries); + REQUIRE(logger->error_count() == 0); + REQUIRE(span); + REQUIRE(span->trace_id().hex_padded() == + "000000000000beef0000000000000004"); + } + + SECTION("extracted from B3") { + std::unordered_map headers; + headers["x-b3-traceid"] = "deadbeefdeadbeefcafebabecafebabe"; + headers["x-b3-spanid"] = "42"; + MockDictReader reader{headers}; + const auto span = tracer.extract_span(reader); + CAPTURE(logger->entries); + REQUIRE(logger->error_count() == 0); + REQUIRE(span); + REQUIRE(hex(span->trace_id().high) == "deadbeefdeadbeef"); + } } From 964bd5557dcb30f4797472961d5ab59328f5029c Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 11:03:20 -0500 Subject: [PATCH 40/54] test the injection of 128-bit trace IDs --- src/datadog/tags.cpp | 1 + src/datadog/tags.h | 1 + src/datadog/tracer.cpp | 5 +-- src/datadog/w3c_propagation.cpp | 2 +- test/test_span.cpp | 60 +++++++++++++++++++++++++++++++++ test/test_tracer.cpp | 2 +- 6 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/datadog/tags.cpp b/src/datadog/tags.cpp index db5828b2..513935ee 100644 --- a/src/datadog/tags.cpp +++ b/src/datadog/tags.cpp @@ -29,6 +29,7 @@ const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism"; const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate"; const std::string span_sampling_limit = "_dd.span_sampling.max_per_second"; const std::string w3c_extraction_error = "_dd.w3c_extraction_error"; +const std::string trace_id_high = "_dd.p.tid"; } // namespace internal diff --git a/src/datadog/tags.h b/src/datadog/tags.h index 996cc58b..030007b2 100644 --- a/src/datadog/tags.h +++ b/src/datadog/tags.h @@ -33,6 +33,7 @@ extern const std::string span_sampling_mechanism; extern const std::string span_sampling_rule_rate; extern const std::string span_sampling_limit; extern const std::string w3c_extraction_error; +extern const std::string trace_id_high; } // namespace internal // Return whether the specified `tag_name` is reserved for use internal to this diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index d690feb3..de7819cb 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -46,7 +46,7 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, continue; } - if (key == "_dd.p.tid") { + if (key == tags::internal::trace_id_high) { // _dd.p.tid contains the high 64 bits of the trace ID. auto high = parse_uint64(value, 16); if (auto* error = high.if_error()) { @@ -269,7 +269,8 @@ Span Tracer::create_span(const SpanConfig& config) { std::vector> trace_tags; span_data->trace_id = generator_->trace_id(); if (span_data->trace_id.high) { - trace_tags.emplace_back("_dd.p.tid", hex(span_data->trace_id.high)); + trace_tags.emplace_back(tags::internal::trace_id_high, + hex(span_data->trace_id.high)); } span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index a6c6132e..2957f735 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -327,7 +327,7 @@ std::string encode_datadog_tracestate( for (const auto& [key, value] : trace_tags) { const StringView prefix = "_dd.p."; - if (!starts_with(key, prefix) || key == "_dd.p.tid") { + if (!starts_with(key, prefix) || key == tags::internal::trace_id_high) { // Either it's not a propagation tag, or it's one of the propagation tags // that need not be included in tracestate. continue; diff --git a/test/test_span.cpp b/test/test_span.cpp index 5cbcf76a..9e1e456b 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -689,3 +689,63 @@ TEST_CASE("injecting W3C tracestate header") { REQUIRE(logger->error_count() == 0); } + +TEST_CASE("128-bit trace ID injection") { + TracerConfig config; + config.defaults.service = "testsvc"; + config.logger = std::make_shared(); + config.trace_id_128_bit = true; + config.injection_styles.clear(); + config.injection_styles.push_back(PropagationStyle::W3C); + config.injection_styles.push_back(PropagationStyle::DATADOG); + config.injection_styles.push_back(PropagationStyle::B3); + + const auto finalized = finalize_config(config); + REQUIRE(finalized); + + class MockIDGenerator : public IDGenerator { + const TraceID trace_id_; + + public: + explicit MockIDGenerator(TraceID trace_id) : trace_id_(trace_id) {} + TraceID trace_id() const override { return trace_id_; } + // `span_id` won't be called, because root spans use the lower part of + // `trace_id` for the span ID. + std::uint64_t span_id() const override { return 42; } + }; + + const auto maybe_trace_id = + TraceID::parse_hex("deadbeefdeadbeefcafebabecafebabe"); + REQUIRE(maybe_trace_id); + const TraceID trace_id = *maybe_trace_id; + const auto expected_low = 14627333968358193854ULL; + const auto expected_high = 16045690984833335023ULL; + REQUIRE(trace_id.low == expected_low); + REQUIRE(trace_id.high == expected_high); + Tracer tracer{*finalized, std::make_shared(trace_id)}; + + auto span = tracer.create_span(); + span.trace_segment().override_sampling_priority(2); + MockDictWriter writer; + span.inject(writer); + + // PropagationStyle::DATADOG + auto found = writer.items.find("x-datadog-trace-id"); + REQUIRE(found != writer.items.end()); + REQUIRE(found->second == std::to_string(expected_low)); + found = writer.items.find("x-datadog-tags"); + REQUIRE(found != writer.items.end()); + REQUIRE(found->second.find("_dd.p.tid=deadbeefdeadbeef") != + std::string::npos); + + // PropagationStyle::W3C + found = writer.items.find("traceparent"); + REQUIRE(found != writer.items.end()); + REQUIRE(found->second == + "00-deadbeefdeadbeefcafebabecafebabe-cafebabecafebabe-01"); + + // PropagationStyle::B3 + found = writer.items.find("x-b3-traceid"); + REQUIRE(found != writer.items.end()); + REQUIRE(found->second == "deadbeefdeadbeefcafebabecafebabe"); +} diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index d3a22ff9..6865b558 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -971,7 +971,7 @@ TEST_CASE("128-bit trace IDs") { REQUIRE(logger->error_count() == 0); REQUIRE(collector->span_count() == 1); const auto& span = collector->first_span(); - const auto found = span.tags.find("_dd.p.tid"); + const auto found = span.tags.find(tags::internal::trace_id_high); REQUIRE(found != span.tags.end()); const auto high = parse_uint64(found->second, 16); REQUIRE(high); From f8a4c6b78ecb5cb4bbf37580051efb4cc41a7683 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 11:17:48 -0500 Subject: [PATCH 41/54] test the configuration of 128-bit trace IDs --- test/test_tracer.cpp | 2 +- test/test_tracer_config.cpp | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 6865b558..aa242670 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -939,7 +939,7 @@ TEST_CASE("report hostname") { } } -TEST_CASE("128-bit trace IDs") { +TEST_CASE("create 128-bit trace IDs") { TracerConfig config; config.defaults.service = "testsvc"; config.trace_id_128_bit = true; diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 4bf7a89b..2c5ae805 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -1195,3 +1195,54 @@ TEST_CASE("TracerConfig propagation styles") { } } } + +TEST_CASE("configure 128-bit trace IDs") { + TracerConfig config; + config.defaults.service = "testsvc"; + + SECTION("defaults to false") { REQUIRE(config.trace_id_128_bit == false); } + + SECTION("value honored in finalizer") { + const auto value = GENERATE(true, false); + config.trace_id_128_bit = value; + const auto finalized = finalize_config(config); + REQUIRE(finalized); + REQUIRE(finalized->trace_id_128_bit == value); + } + + SECTION("value overridden by DD_TRACE_ID_128_BIT_ENABLED") { + struct TestCase { + int line; + std::string env_value; + bool expected_value; + }; + + // clang-format off + const auto test_case = GENERATE(values({ + {__LINE__, "true", true}, + {__LINE__, "false", false}, + {__LINE__, "no", false}, + {__LINE__, "nein", true}, + {__LINE__, "0", false}, + {__LINE__, "", true}, + })); + // clang-format on + + CAPTURE(test_case.line); + CAPTURE(test_case.env_value); + + EnvGuard guard{"DD_TRACE_ID_128_BIT_ENABLED", test_case.env_value}; + + config.trace_id_128_bit = true; + CAPTURE(config.trace_id_128_bit); + auto finalized = finalize_config(config); + REQUIRE(finalized); + REQUIRE(finalized->trace_id_128_bit == test_case.expected_value); + + config.trace_id_128_bit = false; + CAPTURE(config.trace_id_128_bit); + finalized = finalize_config(config); + REQUIRE(finalized); + REQUIRE(finalized->trace_id_128_bit == test_case.expected_value); + } +} From 758bfbf5a1be9c6e2afbfbc72a21ae6371aa6a1d Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 11:22:20 -0500 Subject: [PATCH 42/54] whoops --- src/datadog/environment.h | 2 +- src/datadog/tracer_config.cpp | 3 ++- test/test_tracer_config.cpp | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/datadog/environment.h b/src/datadog/environment.h index 20f65f7c..adf06fae 100644 --- a/src/datadog/environment.h +++ b/src/datadog/environment.h @@ -47,7 +47,7 @@ namespace environment { MACRO(DD_TRACE_STARTUP_LOGS) \ MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ MACRO(DD_VERSION) \ - MACRO(DD_TRACE_ID_128_BIT_ENABLED) + MACRO(DD_TRACE_128_BIT_TRACEID_ENABLED) #define WITH_COMMA(ARG) ARG, diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 3011ce02..3029b3d9 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -360,7 +360,8 @@ Expected finalize_config(const TracerConfig &config) { result.report_hostname = config.report_hostname; result.tags_header_size = config.tags_header_size; - if (auto enabled_env = lookup(environment::DD_TRACE_ID_128_BIT_ENABLED)) { + if (auto enabled_env = + lookup(environment::DD_TRACE_128_BIT_TRACEID_ENABLED)) { result.trace_id_128_bit = !falsy(*enabled_env); } else { result.trace_id_128_bit = config.trace_id_128_bit; diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 2c5ae805..590795dc 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -1210,7 +1210,7 @@ TEST_CASE("configure 128-bit trace IDs") { REQUIRE(finalized->trace_id_128_bit == value); } - SECTION("value overridden by DD_TRACE_ID_128_BIT_ENABLED") { + SECTION("value overridden by DD_TRACE_128_BIT_TRACEID_ENABLED") { struct TestCase { int line; std::string env_value; @@ -1231,7 +1231,7 @@ TEST_CASE("configure 128-bit trace IDs") { CAPTURE(test_case.line); CAPTURE(test_case.env_value); - EnvGuard guard{"DD_TRACE_ID_128_BIT_ENABLED", test_case.env_value}; + EnvGuard guard{"DD_TRACE_128_BIT_TRACEID_ENABLED", test_case.env_value}; config.trace_id_128_bit = true; CAPTURE(config.trace_id_128_bit); From b45b1544a8fb78f699fc74529f1192616ee0e2b2 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 11:25:06 -0500 Subject: [PATCH 43/54] remove spurious TODO --- src/datadog/trace_id.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index 10c38aa0..bd9c1271 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -14,7 +14,6 @@ TraceID::TraceID(std::uint64_t low, std::uint64_t high) : low(low), high(high) {} std::string TraceID::hex_padded() const { - // TODO std::string result; if (high) { result += ::datadog::tracing::hex_padded(high); From ea014b66f10e43365914c827db53fcb4315815bd Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 16 Jan 2023 15:20:26 -0500 Subject: [PATCH 44/54] test TraceID::parse_hex --- src/datadog/trace_id.cpp | 2 +- test/CMakeLists.txt | 1 + test/test_trace_id.cpp | 51 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 test/test_trace_id.cpp diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index bd9c1271..36a37266 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -33,7 +33,7 @@ std::string TraceID::debug() const { Expected TraceID::parse_hex(StringView input) { const auto parse_hex_piece = - [&](StringView piece) -> Expected { + [input](StringView piece) -> Expected { auto result = parse_uint64(piece, 16); if (auto *error = result.if_error()) { std::string prefix = "Unable to parse trace ID from \""; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 928c152f..84b7dc9e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(tests test_smoke.cpp test_span.cpp test_span_sampler.cpp + test_trace_id.cpp test_trace_segment.cpp test_tracer_config.cpp test_tracer.cpp diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp new file mode 100644 index 00000000..77ab33c9 --- /dev/null +++ b/test/test_trace_id.cpp @@ -0,0 +1,51 @@ +// This test covers operations defined for `class TraceID` in `trace_id.h`. + +#include +#include +#include + +#include "test.h" + +using namespace datadog::tracing; + +TEST_CASE("TraceID defaults to zero") { + TraceID id; + REQUIRE(id.low == 0); + REQUIRE(id.high == 0); +} + +TEST_CASE("TraceID parsed from hexadecimal") { + // TODO: leading zeroes might be a corner case. + struct TestCase { + int line; + std::string input; + Optional expected_id; + Optional expected_error = nullopt; + }; + + // clang-format off + const auto test_case = GENERATE(values({ + {__LINE__, "00001", TraceID(1)}, + {__LINE__, "0000000000000000000000000000000000000000000001", TraceID(1)}, + {__LINE__, "", nullopt, Error::INVALID_INTEGER}, + {__LINE__, "nonsense", nullopt, Error::INVALID_INTEGER}, + {__LINE__, "1000000000000000000000000000000000000000000000", nullopt, Error::OUT_OF_RANGE_INTEGER}, + {__LINE__, "deadbeefdeadbeef", TraceID{0xdeadbeefdeadbeefULL}}, + {__LINE__, "0xdeadbeefdeadbeef", nullopt, Error::INVALID_INTEGER}, + {__LINE__, "cafebabecafebabedeadbeefdeadbeef", TraceID{0xdeadbeefdeadbeefULL, 0xcafebabecafebabe}}, + {__LINE__, "caxxxxxxcafebabedeadbeefdeadbeef", nullopt, Error::INVALID_INTEGER}, + {__LINE__, "cafebabecafebabedeaxxxxxxxxdbeef", nullopt, Error::INVALID_INTEGER}, + })); + // clang-format on + + CAPTURE(test_case.line); + CAPTURE(test_case.input); + const auto result = TraceID::parse_hex(test_case.input); + if (test_case.expected_error) { + REQUIRE_FALSE(result); + REQUIRE(result.error().code == *test_case.expected_error); + } else { + REQUIRE(result); + REQUIRE(*result == *test_case.expected_id); + } +} From 036c94f6609f342b1e1d38bad7e8faf53e418af6 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 09:01:26 -0500 Subject: [PATCH 45/54] remove unnecessary parsing in test --- test/test_span.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/test/test_span.cpp b/test/test_span.cpp index 9e1e456b..29c2f0c6 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -714,14 +714,7 @@ TEST_CASE("128-bit trace ID injection") { std::uint64_t span_id() const override { return 42; } }; - const auto maybe_trace_id = - TraceID::parse_hex("deadbeefdeadbeefcafebabecafebabe"); - REQUIRE(maybe_trace_id); - const TraceID trace_id = *maybe_trace_id; - const auto expected_low = 14627333968358193854ULL; - const auto expected_high = 16045690984833335023ULL; - REQUIRE(trace_id.low == expected_low); - REQUIRE(trace_id.high == expected_high); + const TraceID trace_id{0xcafebabecafebabeULL, 0xdeadbeefdeadbeefULL}; Tracer tracer{*finalized, std::make_shared(trace_id)}; auto span = tracer.create_span(); @@ -732,7 +725,7 @@ TEST_CASE("128-bit trace ID injection") { // PropagationStyle::DATADOG auto found = writer.items.find("x-datadog-trace-id"); REQUIRE(found != writer.items.end()); - REQUIRE(found->second == std::to_string(expected_low)); + REQUIRE(found->second == std::to_string(trace_id.low)); found = writer.items.find("x-datadog-tags"); REQUIRE(found != writer.items.end()); REQUIRE(found->second.find("_dd.p.tid=deadbeefdeadbeef") != From c0cd83d157ecb65805435721642da046ff99aa53 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 09:17:06 -0500 Subject: [PATCH 46/54] test TraceID comparisons --- test/test_trace_id.cpp | 47 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp index 77ab33c9..7d5f059f 100644 --- a/test/test_trace_id.cpp +++ b/test/test_trace_id.cpp @@ -4,14 +4,24 @@ #include #include +#include + #include "test.h" using namespace datadog::tracing; +std::ostream& operator<<(std::ostream& stream, TraceID trace_id) { + return stream << trace_id.debug(); +} + TEST_CASE("TraceID defaults to zero") { - TraceID id; - REQUIRE(id.low == 0); - REQUIRE(id.high == 0); + TraceID id1; + REQUIRE(id1.low == 0); + REQUIRE(id1.high == 0); + + TraceID id2{0xdeadbeef}; + REQUIRE(id2.low == 0xdeadbeef); + REQUIRE(id2.high == 0); } TEST_CASE("TraceID parsed from hexadecimal") { @@ -49,3 +59,34 @@ TEST_CASE("TraceID parsed from hexadecimal") { REQUIRE(*result == *test_case.expected_id); } } + +TEST_CASE("TraceID comparisons") { + struct TestCase { + int line; + std::string name; + TraceID left; + TraceID right; + bool equal; + }; + + // clang-format off + const auto test_case = GENERATE(values({ + {__LINE__, "defaults", TraceID{}, TraceID{}, true}, + {__LINE__, "lowers equal", TraceID{0xcafebabe}, TraceID{0xcafebabe}, true}, + {__LINE__, "lowers not equal", TraceID{0xcafebabe}, TraceID{0xdeadbeef}, false}, + {__LINE__, "highers zeroness agree", TraceID{0xcafebabe, 0xdeadbeef}, TraceID{0xcafebabe, 0xdeadbeef}, true}, + {__LINE__, "highers zeroness disagree", TraceID{0xdeadbeef}, TraceID{0xcafebabe, 0xdeadbeef}, false}, + {__LINE__, "highers disagree", TraceID{0xdeadbeef, 0xdeadbeef}, TraceID{0xcafebabe, 0xdeadbeef}, false}, + })); + // clang-format on + + CAPTURE(test_case.line); + CAPTURE(test_case.name); + if (test_case.equal) { + REQUIRE(test_case.left == test_case.right); + REQUIRE_FALSE(test_case.left != test_case.right); + } else { + REQUIRE_FALSE(test_case.left == test_case.right); + REQUIRE(test_case.left != test_case.right); + } +} From 1ce7f284192554b7ad97849d9fbebed3ce98b756 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 09:35:59 -0500 Subject: [PATCH 47/54] document trace_id.h --- src/datadog/trace_id.cpp | 4 ++-- src/datadog/trace_id.h | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index 36a37266..3082a9de 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -84,11 +84,11 @@ bool operator!=(TraceID left, TraceID right) { } bool operator==(TraceID left, std::uint64_t right) { - return left.high == 0 && left.low == right; + return left == TraceID{right}; } bool operator!=(TraceID left, std::uint64_t right) { - return left.high != 0 || left.low != right; + return left != TraceID{right}; } } // namespace tracing diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h index bcb21178..7b2be0df 100644 --- a/src/datadog/trace_id.h +++ b/src/datadog/trace_id.h @@ -1,6 +1,9 @@ #pragma once -// TODO: document +// This component provides a `struct`, `TraceID`, that represents an opaque, +// unique identifier for a trace. +// `TraceID` is 128 bits wide, though in some contexts only the lower 64 bits +// are used. #include #include @@ -15,16 +18,29 @@ struct TraceID { std::uint64_t low; std::uint64_t high; - // TODO: document + // Create a zero trace ID. TraceID(); + + // Create a trace ID whose lower 64 bits are the specified `low` and whose + // higher 64 bits are zero. explicit TraceID(std::uint64_t low); + + // Create a trace ID whose lower 64 bits are the specified `low` and whose + // higher 64 bits are the specified `high`. TraceID(std::uint64_t low, std::uint64_t high); - // TODO: document + // Return a 32 character lower-case hexadecimal representation of this trace + // ID, padded with zeroes on the left. std::string hex_padded() const; + + // Return either a decimal or a "0x"-prefixed hexadecimal representation of + // this trace ID, depending on whether the higher 64 bits are zero. This + // representation is meant for use in error diagnostics. std::string debug() const; - // TODO: document + // Return a `TraceID` parsed from the specified hexadecimal string, or return + // an `Error`. It is an error of the input contains any non-hexadecimal + // characters. static Expected parse_hex(StringView); }; From 8df560a8c639f8384cb760613e2e2bd80f45d6b7 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 09:42:03 -0500 Subject: [PATCH 48/54] cover the rest of the comparison cases --- test/test_trace_id.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp index 7d5f059f..6e4d9f52 100644 --- a/test/test_trace_id.cpp +++ b/test/test_trace_id.cpp @@ -61,6 +61,15 @@ TEST_CASE("TraceID parsed from hexadecimal") { } TEST_CASE("TraceID comparisons") { + // First, comparing integers with the `TraceID.low`. + REQUIRE(TraceID{12345} == 12345); + REQUIRE_FALSE(TraceID{12345} != 12345); + REQUIRE(TraceID{12345} != 54321); + REQUIRE_FALSE(TraceID{12345} == 54321); + REQUIRE(TraceID{6789, 12345} != 12345); + REQUIRE_FALSE(TraceID{6789, 12345} == 12345); + + // Second, comparing trace IDs with other trace IDs. struct TestCase { int line; std::string name; From 9841db7330d8c57c24e1fded21e9a7996eab06db Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 09:42:28 -0500 Subject: [PATCH 49/54] remove spurious TODO --- test/test_trace_id.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp index 6e4d9f52..51377cf0 100644 --- a/test/test_trace_id.cpp +++ b/test/test_trace_id.cpp @@ -25,7 +25,6 @@ TEST_CASE("TraceID defaults to zero") { } TEST_CASE("TraceID parsed from hexadecimal") { - // TODO: leading zeroes might be a corner case. struct TestCase { int line; std::string input; From 18aead7beefce4dd420742b969488c9f37e57fb5 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 17 Jan 2023 10:01:54 -0500 Subject: [PATCH 50/54] test TraceID serialization --- test/test_trace_id.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp index 51377cf0..3cb7a73a 100644 --- a/test/test_trace_id.cpp +++ b/test/test_trace_id.cpp @@ -98,3 +98,31 @@ TEST_CASE("TraceID comparisons") { REQUIRE(test_case.left != test_case.right); } } + +TEST_CASE("TraceID serialization") { + struct TestCase { + int line; + std::string trace_id_source; + TraceID trace_id; + std::string expected_hex; + std::string expected_debug; + }; + +#define CASE(TRACE_ID, HEX, DEBUG) \ + { __LINE__, #TRACE_ID, TRACE_ID, HEX, DEBUG } + // clang-format off + const auto test_case = GENERATE(values({ + CASE(TraceID(), "00000000000000000000000000000000", "0"), + CASE(TraceID(16), "00000000000000000000000000000010", "16"), + CASE(TraceID(0xcafebabe), "000000000000000000000000cafebabe", "3405691582"), + CASE(TraceID(0, 1), "00000000000000010000000000000000", "0x00000000000000010000000000000000"), + CASE(TraceID(15, 0xcafebabe), "00000000cafebabe000000000000000f", "0x00000000cafebabe000000000000000f"), + })); +// clang-format on +#undef CASE + + CAPTURE(test_case.line); + CAPTURE(test_case.trace_id_source); + REQUIRE(test_case.trace_id.hex_padded() == test_case.expected_hex); + REQUIRE(test_case.trace_id.debug() == test_case.expected_debug); +} From db9e1dad9939801abb1b8d8921181a2085fa501e Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 18 Jan 2023 20:47:46 -0500 Subject: [PATCH 51/54] document what knuth_hash is for, mentioning 128-bit trace IDs --- src/datadog/sampling_util.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/datadog/sampling_util.h b/src/datadog/sampling_util.h index bf9c6921..825484ae 100644 --- a/src/datadog/sampling_util.h +++ b/src/datadog/sampling_util.h @@ -11,6 +11,16 @@ namespace datadog { namespace tracing { +// Return a hash value for the specified `value`. `value` is one of the +// following: +// +// - a 64-bit span ID +// - a 64-bit trace ID +// - the lower 64 bits of a 128-bit trace ID +// +// The resulting hash value is compared with an upper bound provided by +// `max_id_from_rate` (below) to determine whether the span/trace associated +// with `value` is eligible for keeping on statistical grounds. inline std::uint64_t knuth_hash(std::uint64_t value) { return value * UINT64_C(1111111111111111111); } From fde9d5770859d86ff22d77c9780333f87cd715bf Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 19 Jan 2023 09:28:54 -0500 Subject: [PATCH 52/54] config option supports 128-bit _generation_, not propagation --- src/datadog/environment.h | 2 +- src/datadog/tracer_config.cpp | 2 +- src/datadog/tracer_config.h | 12 ++++-------- test/test_tracer_config.cpp | 5 +++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/datadog/environment.h b/src/datadog/environment.h index adf06fae..472bada9 100644 --- a/src/datadog/environment.h +++ b/src/datadog/environment.h @@ -47,7 +47,7 @@ namespace environment { MACRO(DD_TRACE_STARTUP_LOGS) \ MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \ MACRO(DD_VERSION) \ - MACRO(DD_TRACE_128_BIT_TRACEID_ENABLED) + MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED) #define WITH_COMMA(ARG) ARG, diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 3029b3d9..48ee55cc 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -361,7 +361,7 @@ Expected finalize_config(const TracerConfig &config) { result.tags_header_size = config.tags_header_size; if (auto enabled_env = - lookup(environment::DD_TRACE_128_BIT_TRACEID_ENABLED)) { + lookup(environment::DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)) { result.trace_id_128_bit = !falsy(*enabled_env); } else { result.trace_id_128_bit = config.trace_id_128_bit; diff --git a/src/datadog/tracer_config.h b/src/datadog/tracer_config.h index a64a2c95..b517723b 100644 --- a/src/datadog/tracer_config.h +++ b/src/datadog/tracer_config.h @@ -98,14 +98,10 @@ struct TracerConfig { // variable. bool log_on_startup = true; - // `trace_id_128_bit` indicates whether the tracer will generate and interpret - // the higher 64 bits of trace IDs. If true, the tracer will generate 128-bit - // trace IDs, extract them from incoming trace context, inject them into - // outgoing trace context, and send them to the collector. If false, then the - // tracer will generate 64-bit trace IDs, and extract only the lower 64 bits - // of trace IDs from incoming trace context. - // `trace_id_128_bit` is overridden by the `DD_TRACE_128_BIT_TRACEID_ENABLED` - // environment variable. + // `trace_id_128_bit` indicates whether the tracer will generate 128-bit trace + // IDs. If true, the tracer will generate 128-bit trace IDs. If false, the + // tracer will generate 64-bit trace IDs. `trace_id_128_bit` is overridden by + // the `DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED` environment variable. bool trace_id_128_bit = false; }; diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 590795dc..e354cc4d 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -1210,7 +1210,7 @@ TEST_CASE("configure 128-bit trace IDs") { REQUIRE(finalized->trace_id_128_bit == value); } - SECTION("value overridden by DD_TRACE_128_BIT_TRACEID_ENABLED") { + SECTION("value overridden by DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") { struct TestCase { int line; std::string env_value; @@ -1231,7 +1231,8 @@ TEST_CASE("configure 128-bit trace IDs") { CAPTURE(test_case.line); CAPTURE(test_case.env_value); - EnvGuard guard{"DD_TRACE_128_BIT_TRACEID_ENABLED", test_case.env_value}; + EnvGuard guard{"DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", + test_case.env_value}; config.trace_id_128_bit = true; CAPTURE(config.trace_id_128_bit); From 028af2c2013e9c510ecd83845607bd2bc78566d9 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 19 Jan 2023 09:38:34 -0500 Subject: [PATCH 53/54] exclude _dd.p.tid from extracted W3C tracestate --- src/datadog/w3c_propagation.cpp | 6 ++++++ test/test_tracer.cpp | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 2957f735..afa265b5 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -214,6 +214,12 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { const auto tag_suffix = key.substr(2); std::string tag_name = "_dd.p."; append(tag_name, tag_suffix); + // The "_dd.p.tid" trace tag is ignored in this context, since its value + // is better inferred from the higher 64 bits of the trace ID. + if (tag_name == "_dd.p.tid") { + pair_begin = pair_end == end ? end : pair_end + 1; + continue; + } // The tag value was encoded with all '=' replaced by '~'. Undo that // transformation. std::string decoded_value{value}; diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index aa242670..3bf09902 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -800,6 +800,15 @@ TEST_CASE("span extraction") { nullopt, // expected_additional_w3c_tracestate "x:wow;y:wow"}, // expected_additional_datadog_w3c_tracestate + {__LINE__, "_dd.p.tid trace tag is ignored", + traceparent_drop, // traceparent + "dd=t.tid:deadbeef;t.foo:bar", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {{"_dd.p.foo", "bar"}}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + {__LINE__, "traceparent and tracestate sampling agree (1/4)", traceparent_drop, // traceparent "dd=s:0", // tracestate From 02dcf130a44ecfecbd45da1c06dc2ca080382ea5 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 19 Jan 2023 09:48:47 -0500 Subject: [PATCH 54/54] remove TraceID::debug() --- src/datadog/trace_id.cpp | 7 ------- src/datadog/trace_id.h | 5 ----- src/datadog/tracer.cpp | 8 +++++++- test/test.cpp | 2 +- test/test_trace_id.cpp | 22 +++++++--------------- 5 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/datadog/trace_id.cpp b/src/datadog/trace_id.cpp index 3082a9de..7f0ead9d 100644 --- a/src/datadog/trace_id.cpp +++ b/src/datadog/trace_id.cpp @@ -24,13 +24,6 @@ std::string TraceID::hex_padded() const { return result; } -std::string TraceID::debug() const { - if (high) { - return "0x" + hex_padded(); - } - return std::to_string(low); -} - Expected TraceID::parse_hex(StringView input) { const auto parse_hex_piece = [input](StringView piece) -> Expected { diff --git a/src/datadog/trace_id.h b/src/datadog/trace_id.h index 7b2be0df..d0e39a5c 100644 --- a/src/datadog/trace_id.h +++ b/src/datadog/trace_id.h @@ -33,11 +33,6 @@ struct TraceID { // ID, padded with zeroes on the left. std::string hex_padded() const; - // Return either a decimal or a "0x"-prefixed hexadecimal representation of - // this trace ID, depending on whether the higher 64 bits are zero. This - // representation is meant for use in error diagnostics. - std::string debug() const; - // Return a `TraceID` parsed from the specified hexadecimal string, or return // an `Error`. It is an error of the input contains any non-hexadecimal // characters. diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index de7819cb..45cece9d 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -363,7 +363,13 @@ Expected Tracer::extract_span(const DictReader& reader, std::string message; message += "There's no parent span ID to extract, but there is a trace ID: "; - message += trace_id->debug(); + message += "[hexadecimal = "; + message += trace_id->hex_padded(); + if (trace_id->high == 0) { + message += ", decimal = "; + message += std::to_string(trace_id->low); + } + message += ']'; return Error{Error::MISSING_PARENT_SPAN_ID, std::move(message)}; } diff --git a/test/test.cpp b/test/test.cpp index 6359577d..6c0e2dd0 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -28,7 +28,7 @@ namespace datadog { namespace tracing { std::ostream& operator<<(std::ostream& stream, TraceID trace_id) { - return stream << trace_id.debug(); + return stream << "0x" << trace_id.hex_padded(); } } // namespace tracing diff --git a/test/test_trace_id.cpp b/test/test_trace_id.cpp index 3cb7a73a..5c2e2bb6 100644 --- a/test/test_trace_id.cpp +++ b/test/test_trace_id.cpp @@ -4,16 +4,10 @@ #include #include -#include - #include "test.h" using namespace datadog::tracing; -std::ostream& operator<<(std::ostream& stream, TraceID trace_id) { - return stream << trace_id.debug(); -} - TEST_CASE("TraceID defaults to zero") { TraceID id1; REQUIRE(id1.low == 0); @@ -105,18 +99,17 @@ TEST_CASE("TraceID serialization") { std::string trace_id_source; TraceID trace_id; std::string expected_hex; - std::string expected_debug; }; -#define CASE(TRACE_ID, HEX, DEBUG) \ - { __LINE__, #TRACE_ID, TRACE_ID, HEX, DEBUG } +#define CASE(TRACE_ID, HEX) \ + { __LINE__, #TRACE_ID, TRACE_ID, HEX } // clang-format off const auto test_case = GENERATE(values({ - CASE(TraceID(), "00000000000000000000000000000000", "0"), - CASE(TraceID(16), "00000000000000000000000000000010", "16"), - CASE(TraceID(0xcafebabe), "000000000000000000000000cafebabe", "3405691582"), - CASE(TraceID(0, 1), "00000000000000010000000000000000", "0x00000000000000010000000000000000"), - CASE(TraceID(15, 0xcafebabe), "00000000cafebabe000000000000000f", "0x00000000cafebabe000000000000000f"), + CASE(TraceID(), "00000000000000000000000000000000"), + CASE(TraceID(16), "00000000000000000000000000000010"), + CASE(TraceID(0xcafebabe), "000000000000000000000000cafebabe"), + CASE(TraceID(0, 1), "00000000000000010000000000000000"), + CASE(TraceID(15, 0xcafebabe), "00000000cafebabe000000000000000f"), })); // clang-format on #undef CASE @@ -124,5 +117,4 @@ TEST_CASE("TraceID serialization") { CAPTURE(test_case.line); CAPTURE(test_case.trace_id_source); REQUIRE(test_case.trace_id.hex_padded() == test_case.expected_hex); - REQUIRE(test_case.trace_id.debug() == test_case.expected_debug); }