From 68850b3a4e4ed8cf39b2b2ad06aa58800846407e Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 14:50:08 -0500 Subject: [PATCH 01/29] TODO: AMEND: add W3C style, but it doesn't do anything --- src/datadog/propagation_style.cpp | 6 +++++- src/datadog/propagation_style.h | 2 ++ src/datadog/trace_segment.cpp | 6 +++++- src/datadog/tracer.cpp | 10 ++++++++++ src/datadog/tracer_config.cpp | 5 +++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/datadog/propagation_style.cpp b/src/datadog/propagation_style.cpp index 146c6cd1..ca4beb8b 100644 --- a/src/datadog/propagation_style.cpp +++ b/src/datadog/propagation_style.cpp @@ -8,11 +8,15 @@ namespace datadog { namespace tracing { nlohmann::json to_json(PropagationStyle style) { + // Note: Make sure that these strings are consistent (modulo case) with + // `parse_propagation_styles` in `tracer_config.cpp`. switch (style) { case PropagationStyle::DATADOG: - return "datadog"; + return "Datadog"; case PropagationStyle::B3: return "B3"; + case PropagationStyle::W3C: + return "tracecontext"; // for compatibility with OpenTelemetry default: assert(style == PropagationStyle::NONE); return "none"; diff --git a/src/datadog/propagation_style.h b/src/datadog/propagation_style.h index 66d29786..ed32efcc 100644 --- a/src/datadog/propagation_style.h +++ b/src/datadog/propagation_style.h @@ -17,6 +17,8 @@ enum class PropagationStyle { DATADOG, // B3 multi-header style, e.g. X-B3-TraceID B3, + // W3C headers style, i.e. traceparent and tracestate + W3C, // The absence of propagation. If this is the only style set, then // propagation is disabled in the relevant direction (extraction or // injection). diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index ac70d96e..878a3147 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -261,7 +261,11 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); break; - case PropagationStyle::NONE: + case PropagationStyle::W3C: + // TODO + break; + default: + assert(style == PropagationStyle::NONE); break; } } diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 81f3bf25..2fbb365d 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -148,6 +148,13 @@ Expected extract_b3(const DictReader& headers) { return result; } +Expected extract_w3c(const DictReader& headers) { + ExtractedData result; + // TODO + (void)headers; + return result; +} + void log_startup_message(Logger& logger, StringView tracer_version_string, const Collector& collector, const SpanDefaults& defaults, @@ -256,6 +263,9 @@ Expected Tracer::extract_span(const DictReader& reader, case PropagationStyle::B3: extract = &extract_b3; break; + case PropagationStyle::W3C: + extract = &extract_w3c; + break; default: assert(style == PropagationStyle::NONE); extracted_data = ExtractedData{}; diff --git a/src/datadog/tracer_config.cpp b/src/datadog/tracer_config.cpp index 814bbef3..6bf95b1a 100644 --- a/src/datadog/tracer_config.cpp +++ b/src/datadog/tracer_config.cpp @@ -94,10 +94,15 @@ Expected> parse_propagation_styles( for (const StringView &item : parse_list(input)) { auto token = std::string(item); to_lower(token); + // Note: Make sure that these strings are consistent (modulo case) with + // `to_json(PropagationStyle)` in `propagation_style.cpp`. if (token == "datadog") { styles.push_back(PropagationStyle::DATADOG); } else if (token == "b3" || token == "b3multi") { styles.push_back(PropagationStyle::B3); + } else if (token == + "tracecontext") { // for compatibility with OpenTelemetry + styles.push_back(PropagationStyle::W3C); } else if (token == "none") { styles.push_back(PropagationStyle::NONE); } else { From 8238cd5e39ceeb93088119f83b8ed35faf626fd4 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 18:26:04 -0500 Subject: [PATCH 02/29] TODO: AMEND: untested traceparent extraction --- BUILD.bazel | 4 ++ CMakeLists.txt | 4 ++ src/datadog/extracted_data.cpp | 1 + src/datadog/extracted_data.h | 28 ++++++++ src/datadog/tags.cpp | 2 + src/datadog/tags.h | 2 + src/datadog/tracer.cpp | 38 +++++------ src/datadog/w3c_propagation.cpp | 110 ++++++++++++++++++++++++++++++++ src/datadog/w3c_propagation.h | 23 +++++++ 9 files changed, 191 insertions(+), 21 deletions(-) create mode 100644 src/datadog/extracted_data.cpp create mode 100644 src/datadog/extracted_data.h create mode 100644 src/datadog/w3c_propagation.cpp create mode 100644 src/datadog/w3c_propagation.h diff --git a/BUILD.bazel b/BUILD.bazel index 088228a5..290cb07e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,6 +16,7 @@ cc_library( "src/datadog/error.cpp", "src/datadog/event_scheduler.cpp", "src/datadog/expected.cpp", + "src/datadog/extracted_data.cpp", "src/datadog/glob.cpp", "src/datadog/http_client.cpp", "src/datadog/id_generator.cpp", @@ -49,6 +50,7 @@ cc_library( "src/datadog/trace_sampler.cpp", "src/datadog/trace_segment.cpp", "src/datadog/version.cpp", + "src/datadog/w3c_propagation.cpp", ], hdrs = [ "src/datadog/cerr_logger.h", @@ -65,6 +67,7 @@ cc_library( "src/datadog/error.h", "src/datadog/event_scheduler.h", "src/datadog/expected.h", + "src/datadog/extracted_data.h", "src/datadog/glob.h", "src/datadog/http_client.h", "src/datadog/id_generator.h", @@ -100,6 +103,7 @@ cc_library( "src/datadog/trace_sampler.h", "src/datadog/trace_segment.h", "src/datadog/version.h", + "src/datadog/w3c_propagation.h", ], copts = [ "-Wall", diff --git a/CMakeLists.txt b/CMakeLists.txt index b6c842c7..cb42dbe4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/error.cpp src/datadog/event_scheduler.cpp src/datadog/expected.cpp + src/datadog/extracted_data.cpp src/datadog/glob.cpp src/datadog/http_client.cpp src/datadog/id_generator.cpp @@ -103,6 +104,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/trace_sampler.cpp src/datadog/trace_segment.cpp src/datadog/version.cpp + src/datadog/w3c_propagation.cpp ) # This library's public headers are just its source headers. @@ -125,6 +127,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/error.h src/datadog/event_scheduler.h src/datadog/expected.h + src/datadog/extracted_data.h src/datadog/glob.h src/datadog/http_client.h src/datadog/id_generator.h @@ -160,6 +163,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/trace_sampler.h src/datadog/trace_segment.h src/datadog/version.h + src/datadog/w3c_propagation.h ) add_dependencies(dd_trace_cpp curl) diff --git a/src/datadog/extracted_data.cpp b/src/datadog/extracted_data.cpp new file mode 100644 index 00000000..9d02ba2e --- /dev/null +++ b/src/datadog/extracted_data.cpp @@ -0,0 +1 @@ +#include "extracted_data.h" diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h new file mode 100644 index 00000000..96fb468f --- /dev/null +++ b/src/datadog/extracted_data.h @@ -0,0 +1,28 @@ +#pragma once + +// TODO: document + +#include +#include + +#include "optional.h" + +namespace datadog { +namespace tracing { + +struct ExtractedData { + Optional trace_id; + Optional parent_id; + Optional origin; + Optional trace_tags; + Optional sampling_priority; + // TODO: document + Optional full_w3c_trace_id_hex; + // TODO: document + Optional additional_w3c_tracestate; + // TODO: document + Optional additional_datadog_w3c_tracestate; +}; + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/tags.cpp b/src/datadog/tags.cpp index 4a3ebdd0..84b91202 100644 --- a/src/datadog/tags.cpp +++ b/src/datadog/tags.cpp @@ -28,6 +28,8 @@ const std::string agent_sample_rate = "_dd.agent_psr"; const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism"; const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate"; const std::string span_sampling_limit = "_dd.span_sampling.max_per_second"; +const std::string w3c_extraction_error = "_dd.w3c_extraction_error"; +const std::string w3c_injection_error = "_dd.w3c_injection_error"; } // namespace internal diff --git a/src/datadog/tags.h b/src/datadog/tags.h index ad79c866..b1d77c58 100644 --- a/src/datadog/tags.h +++ b/src/datadog/tags.h @@ -32,6 +32,8 @@ extern const std::string agent_sample_rate; extern const std::string span_sampling_mechanism; extern const std::string span_sampling_rule_rate; extern const std::string span_sampling_limit; +extern const std::string w3c_extraction_error; +extern const std::string w3c_injection_error; } // namespace internal // Return whether the specified `tag_name` is reserved for use internal to this diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 2fbb365d..644aa1fb 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -6,6 +6,7 @@ #include "datadog_agent.h" #include "dict_reader.h" #include "environment.h" +#include "extracted_data.h" #include "json.hpp" #include "logger.h" #include "net_util.h" @@ -19,6 +20,7 @@ #include "trace_sampler.h" #include "trace_segment.h" #include "version.h" +#include "w3c_propagation.h" namespace datadog { namespace tracing { @@ -50,15 +52,9 @@ Expected> extract_id_header(const DictReader& headers, return *result; } -struct ExtractedData { - Optional trace_id; - Optional parent_id; - Optional origin; - Optional trace_tags; - Optional sampling_priority; -}; - -Expected extract_datadog(const DictReader& headers) { +Expected extract_datadog( + const DictReader& headers, + std::unordered_map& /*span_tags*/) { ExtractedData result; auto trace_id = @@ -103,7 +99,9 @@ Expected extract_datadog(const DictReader& headers) { return result; } -Expected extract_b3(const DictReader& headers) { +Expected extract_b3( + const DictReader& headers, + std::unordered_map& /*span_tags*/) { ExtractedData result; auto trace_id = extract_id_header(headers, "x-b3-traceid", "trace", "B3", 16); @@ -148,13 +146,6 @@ Expected extract_b3(const DictReader& headers) { return result; } -Expected extract_w3c(const DictReader& headers) { - ExtractedData result; - // TODO - (void)headers; - return result; -} - void log_startup_message(Logger& logger, StringView tracer_version_string, const Collector& collector, const SpanDefaults& defaults, @@ -251,6 +242,7 @@ Expected Tracer::extract_span(const DictReader& reader, const SpanConfig& config) { assert(!extraction_styles_.empty()); + auto span_data = std::make_unique(); ExtractedData extracted_data; for (const auto style : extraction_styles_) { @@ -271,7 +263,7 @@ Expected Tracer::extract_span(const DictReader& reader, extracted_data = ExtractedData{}; continue; } - auto data = extract(reader); + auto data = extract(reader, span_data->tags); if (auto* error = data.if_error()) { return std::move(*error); } @@ -284,8 +276,13 @@ Expected Tracer::extract_span(const DictReader& reader, } } - auto& [trace_id, parent_id, origin, trace_tags, sampling_priority] = - extracted_data; + auto& [trace_id, parent_id, origin, trace_tags, sampling_priority, + full_w3c_trace_id_hex, additional_w3c_tracestate, + additional_datadog_w3c_tracestate] = extracted_data; + + (void)full_w3c_trace_id_hex; // TODO + (void)additional_w3c_tracestate; // TODO + (void)additional_datadog_w3c_tracestate; // TODO // Some information might be missing. // Here are the combinations considered: @@ -332,7 +329,6 @@ Expected Tracer::extract_span(const DictReader& reader, assert(parent_id); assert(trace_id); - auto span_data = std::make_unique(); span_data->apply_config(*defaults_, config, clock_); span_data->span_id = generator_(); span_data->trace_id = *trace_id; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp new file mode 100644 index 00000000..88d9c269 --- /dev/null +++ b/src/datadog/w3c_propagation.cpp @@ -0,0 +1,110 @@ +#include "w3c_propagation.h" + +#include +#include +#include +#include + +#include "dict_reader.h" +#include "parse_util.h" +#include "tags.h" + +namespace datadog { +namespace tracing { +namespace { + +// TODO: document +Optional extract_traceparent(ExtractedData& result, + const DictReader& headers) { + const auto maybe_traceparent = headers.lookup("traceparent"); + if (!maybe_traceparent) { + return nullopt; + } + + const auto traceparent = strip(*maybe_traceparent); + + // Note that leading and trailing whitespace was already removed above. + // Note that the "zero'th" match group is the entire match. + static const auto& pattern = + "([0-9a-f]{2})" // hex version number (match group 1) + "-" + "([0-9a-f]{16}([0-9a-f]{16}))" // hex trace ID (match groups 2 and 3) + "-" + "([0-9a-f]{16})" // hex parent span ID (match group 4) + "-" + "([0-9a-f]{2})" // hex "trace-flags" (match group 5) + "(?:$|-.*)"; // either the end, or a hyphen preceding further fields + thread_local const std::regex regex{pattern}; + + std::match_results match; + if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { + return "malformed_traceparent"; + } + + assert(match.ready()); + assert(match.size() == 5 + 1); + + const auto to_string_view = [](const auto& submatch) { + assert(submatch.first <= submatch.second); + return StringView{submatch.first, + std::size_t(submatch.second - submatch.first)}; + }; + + result.full_w3c_trace_id_hex = std::string{to_string_view(match[2])}; + if (result.full_w3c_trace_id_hex->find_first_not_of('0') == + std::string::npos) { + return "trace_id_zero"; + } + + result.trace_id = *parse_uint64(to_string_view(match[3]), 16); + + result.parent_id = *parse_uint64(to_string_view(match[4]), 16); + if (*result.parent_id == 0) { + return "parent_id_zero"; + } + + const auto flags = *parse_uint64(to_string_view(match[5]), 16); + result.sampling_priority = int(flags & 1); + + return nullopt; +} + +// TODO: document +Optional extract_tracestate(ExtractedData& result, + const DictReader& headers) { + // TODO + (void)result; + (void)headers; + return nullopt; +} + +} // namespace + +Expected extract_w3c( + const DictReader& headers, + std::unordered_map& span_tags) { + ExtractedData result; + + if (auto error_tag_value = extract_traceparent(result, headers)) { + span_tags[tags::internal::w3c_extraction_error] = + std::move(*error_tag_value); + return ExtractedData{}; + } + + // If we didn't get a trace ID from traceparent, don't bother with + // tracestate. + if (!result.trace_id) { + return result; + } + + if (auto error_tag_value = extract_tracestate(result, headers)) { + span_tags[tags::internal::w3c_extraction_error] = + std::move(*error_tag_value); + // Carry on with whatever data was extracted from traceparent. + } + + return result; +} + +} // namespace tracing +} // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h new file mode 100644 index 00000000..6be7d6f8 --- /dev/null +++ b/src/datadog/w3c_propagation.h @@ -0,0 +1,23 @@ +#pragma once + +// TODO: document + +#include +#include + +#include "expected.h" +#include "extracted_data.h" +#include "optional.h" + +namespace datadog { +namespace tracing { + +class DictReader; + +// TODO: document +Expected extract_w3c( + const DictReader& headers, + std::unordered_map& span_tags); + +} // namespace tracing +} // namespace datadog From 1a06e865ea1ab8d9d647426bcdb55ec48e1cb1f1 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 19:46:26 -0500 Subject: [PATCH 03/29] TODO: AMEND: it's already thread-safe --- src/datadog/w3c_propagation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 88d9c269..8e517c8c 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -34,7 +34,7 @@ Optional extract_traceparent(ExtractedData& result, "-" "([0-9a-f]{2})" // hex "trace-flags" (match group 5) "(?:$|-.*)"; // either the end, or a hyphen preceding further fields - thread_local const std::regex regex{pattern}; + static const std::regex regex{pattern}; std::match_results match; if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { From 122fc3d148ee600b51e130859e3b52ef2653eebb Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 19:50:34 -0500 Subject: [PATCH 04/29] add a convenience script that does some of the CI checks --- bin/check | 13 +++++++++++++ bin/format | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 bin/check diff --git a/bin/check b/bin/check new file mode 100755 index 00000000..2d5b4fde --- /dev/null +++ b/bin/check @@ -0,0 +1,13 @@ +#!/bin/sh + +# Run some of the checks that are performed by CircleCI. +# This is convenient to run before committing. + +set -e + +# Go to the repository root directory. +cd "$(dirname "$0")"/.. + +bin/format --dry-run -Werror +bin/test +bin/bazel-build diff --git a/bin/format b/bin/format index 119f7536..5efa9fd9 100755 --- a/bin/format +++ b/bin/format @@ -9,7 +9,7 @@ cd "$(dirname "$0")"/.. # occasionally bumps the required version, reformatting everything. version=14 formatter=clang-format-$version -formatter_options='--style=file -i' +formatter_options="--style=file -i $@" find_sources() { find src/ example/ test/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" From fea21152a5a7ca5f55ca71017848a3f8f6c66d70 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 19 Dec 2022 20:00:04 -0500 Subject: [PATCH 05/29] TODO: AMEND: whoops --- src/datadog/w3c_propagation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 8e517c8c..2b0dfbc9 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -37,7 +37,7 @@ Optional extract_traceparent(ExtractedData& result, static const std::regex regex{pattern}; std::match_results match; - if (!std::regex_match(traceparent.begin(), traceparent.end(), regex)) { + if (!std::regex_match(traceparent.begin(), traceparent.end(), match, regex)) { return "malformed_traceparent"; } From aaea195403aa8c0add7c256275cf693d9771a320 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 18:50:04 -0500 Subject: [PATCH 06/29] TODO: AMEND: extract traceparent --- src/datadog/span_data.cpp | 4 +- src/datadog/trace_segment.cpp | 9 +- src/datadog/trace_segment.h | 6 ++ src/datadog/tracer.cpp | 11 ++- src/datadog/w3c_propagation.cpp | 6 +- test/test.cpp | 2 +- test/test.h | 7 +- test/test_tracer.cpp | 149 +++++++++++++++++++++++++++++++- 8 files changed, 180 insertions(+), 14 deletions(-) diff --git a/src/datadog/span_data.cpp b/src/datadog/span_data.cpp index bf83cc32..fea93cad 100644 --- a/src/datadog/span_data.cpp +++ b/src/datadog/span_data.cpp @@ -38,7 +38,9 @@ void SpanData::apply_config(const SpanDefaults& defaults, service = config.service.value_or(defaults.service); name = config.name.value_or(defaults.name); - tags = defaults.tags; + for (const auto& item : defaults.tags) { + tags.insert(item); + } std::string environment = config.environment.value_or(defaults.environment); if (!environment.empty()) { tags.insert_or_assign(tags::environment, environment); diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 878a3147..254fdba6 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -50,6 +50,9 @@ TraceSegment::TraceSegment( std::size_t tags_header_max_size, std::unordered_map trace_tags, Optional sampling_decision, + Optional full_w3c_trace_id_hex, + Optional additional_w3c_tracestate, + Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root) : logger_(logger), collector_(collector), @@ -62,7 +65,11 @@ TraceSegment::TraceSegment( tags_header_max_size_(tags_header_max_size), trace_tags_(std::move(trace_tags)), num_finished_spans_(0), - sampling_decision_(std::move(sampling_decision)) { + sampling_decision_(std::move(sampling_decision)), + full_w3c_trace_id_hex_(std::move(full_w3c_trace_id_hex)), + additional_w3c_tracestate_(std::move(additional_w3c_tracestate)), + additional_datadog_w3c_tracestate_( + std::move(additional_datadog_w3c_tracestate)) { assert(logger_); assert(collector_); assert(trace_sampler_); diff --git a/src/datadog/trace_segment.h b/src/datadog/trace_segment.h index 0256e405..772e9c36 100644 --- a/src/datadog/trace_segment.h +++ b/src/datadog/trace_segment.h @@ -67,6 +67,9 @@ class TraceSegment { std::vector> spans_; std::size_t num_finished_spans_; Optional sampling_decision_; + Optional full_w3c_trace_id_hex_; + Optional additional_w3c_tracestate_; + Optional additional_datadog_w3c_tracestate_; bool awaiting_delegated_sampling_decision_ = false; public: @@ -80,6 +83,9 @@ class TraceSegment { Optional origin, std::size_t tags_header_max_size, std::unordered_map trace_tags, Optional sampling_decision, + Optional full_w3c_trace_id_hex, + Optional additional_w3c_tracestate, + Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root); const SpanDefaults& defaults() const; diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 644aa1fb..7dae2697 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -229,7 +229,9 @@ Span Tracer::create_span(const SpanConfig& config) { logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, std::unordered_map{} /* trace_tags */, - nullopt /* sampling_decision */, std::move(span_data)); + nullopt /* sampling_decision */, nullopt /* full_w3c_trace_id_hex */, + nullopt /* additional_w3c_tracestate */, + nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; } @@ -280,10 +282,6 @@ Expected Tracer::extract_span(const DictReader& reader, full_w3c_trace_id_hex, additional_w3c_tracestate, additional_datadog_w3c_tracestate] = extracted_data; - (void)full_w3c_trace_id_hex; // TODO - (void)additional_w3c_tracestate; // TODO - (void)additional_datadog_w3c_tracestate; // TODO - // Some information might be missing. // Here are the combinations considered: // @@ -365,7 +363,8 @@ Expected Tracer::extract_span(const DictReader& reader, logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, std::move(origin), tags_header_max_size_, std::move(decoded_trace_tags), std::move(sampling_decision), - std::move(span_data)); + std::move(full_w3c_trace_id_hex), std::move(additional_w3c_tracestate), + std::move(additional_datadog_w3c_tracestate), std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; return span; } diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 2b0dfbc9..7b573460 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -24,7 +24,7 @@ Optional extract_traceparent(ExtractedData& result, const auto traceparent = strip(*maybe_traceparent); // Note that leading and trailing whitespace was already removed above. - // Note that the "zero'th" match group is the entire match. + // Note that the match group 0 is the entire match. static const auto& pattern = "([0-9a-f]{2})" // hex version number (match group 1) "-" @@ -50,6 +50,10 @@ Optional extract_traceparent(ExtractedData& result, std::size_t(submatch.second - submatch.first)}; }; + if (to_string_view(match[1]) == "ff") { + return "invalid_version"; + } + result.full_w3c_trace_id_hex = std::string{to_string_view(match[2])}; if (result.full_w3c_trace_id_hex->find_first_not_of('0') == std::string::npos) { diff --git a/test/test.cpp b/test/test.cpp index 402ae6cc..ca57ed79 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -11,7 +11,7 @@ std::ostream& operator<<( std::ostream& operator<<( std::ostream& stream, const datadog::tracing::Optional& item) { - return stream << item.value_or(""); + return stream << item.value_or(""); } } // namespace std diff --git a/test/test.h b/test/test.h index a0b16615..e9a8664a 100644 --- a/test/test.h +++ b/test/test.h @@ -11,7 +11,9 @@ #include #include +#include #include +#include #include #include "catch.hpp" @@ -21,9 +23,8 @@ namespace std { std::ostream& operator<<(std::ostream& stream, const std::pair& item); -std::ostream& operator<<( - std::ostream& stream, - const datadog::tracing::Optional& item); +std::ostream& operator<<(std::ostream& stream, + const std::optional& maybe); } // namespace std diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 9f0ebdc7..981c2b35 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -239,7 +240,8 @@ TEST_CASE("tracer span defaults") { TEST_CASE("span extraction") { TracerConfig config; config.defaults.service = "testsvc"; - config.collector = std::make_shared(); + const auto collector = std::make_shared(); + config.collector = collector; config.logger = std::make_shared(); SECTION( @@ -504,6 +506,151 @@ TEST_CASE("span extraction") { REQUIRE(result.error().code == Error::NO_SPAN_TO_EXTRACT); } + SECTION("W3C traceparent extraction") { + const std::unordered_map datadog_headers{ + {"x-datadog-trace-id", "18"}, + {"x-datadog-parent-id", "23"}, + {"x-datadog-sampling-priority", "-1"}, + }; + + struct TestCase { + int line; + std::string name; + Optional traceparent; + Optional expected_error_tag_value = {}; + Optional expected_full_trace_id = {}; + Optional expected_trace_id = {}; + Optional expected_parent_id = {}; + Optional expected_sampling_priority = {}; + }; + + // clang-format off + auto test_case = GENERATE(values({ + // https://www.w3.org/TR/trace-context/#examples-of-http-traceparent-headers + {__LINE__, "valid: w3.org example 1", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 1}, // expected_sampling_priority + + {__LINE__, "valid: w3.org example 2", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "valid: future version", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "valid: future version with extra fields", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00-af-delta", // traceparent + nullopt, + "4bf92f3577b34da6a3ce929d0e0e4736", // expected_full_trace_id + 11803532876627986230ULL, // expected_trace_id + 67667974448284343ULL, // expected_parent_id + 0}, // expected_sampling_priority + + {__LINE__, "no traceparent", + nullopt}, // traceparent + + {__LINE__, "invalid: not enough fields", + "06-4bf92f3577b34da6a3ce929d0e0e4736", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: missing hyphen", + "064bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: extra data not preceded by hyphen", + "06-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00af-delta", // traceparent + "malformed_traceparent"}, // expected_error_tag_value + + {__LINE__, "invalid: version", + "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", // traceparent + "invalid_version"}, // expected_error_tag_value + + {__LINE__, "invalid: trace ID zero", + "00-00000000000000000000000000000000-00f067aa0ba902b7-00", // traceparent + "trace_id_zero"}, // expected_error_tag_value + + {__LINE__, "invalid: parent ID zero", + "00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-00", // traceparent + "parent_id_zero"}, // expected_error_tag_value + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + + config.extraction_styles = {PropagationStyle::W3C, + PropagationStyle::DATADOG}; + auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + Tracer tracer{*finalized_config}; + + auto headers = datadog_headers; + if (test_case.traceparent) { + headers["traceparent"] = *test_case.traceparent; + } + MockDictReader reader{headers}; + + // We can't `span->lookup(tags::internal::w3c_extraction_error)`, because + // that tag is internal and will not be returned by `lookup`. Instead, we + // finish (destroy) the span to send it to a collector, and then inspect the + // `SpanData` at the collector. + Optional decision; + { + auto span = tracer.extract_span(reader); + REQUIRE(span); + decision = span->trace_segment().sampling_decision(); + } + + REQUIRE(collector->span_count() == 1); + const auto& span_data = collector->first_span(); + + if (test_case.expected_error_tag_value) { + const auto error_found = + span_data.tags.find(tags::internal::w3c_extraction_error); + REQUIRE(error_found != span_data.tags.end()); + REQUIRE(error_found->second == *test_case.expected_error_tag_value); + // Extraction would have fallen back to the next configured style (Datadog + // -- see `config.extraction_styles`, above), and so the span's properties + // should match `datadog_headers`, above. + REQUIRE(span_data.trace_id == 18); + REQUIRE(span_data.parent_id == 23); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == -1); + } else if (!test_case.traceparent) { + // There was no error extracting W3C context, but there was none to + // extract. + // Extraction would have fallen back to the next configured style (Datadog + // -- see `config.extraction_styles`, above), and so the span's properties + // should match `datadog_headers`, above. + REQUIRE(span_data.trace_id == 18); + REQUIRE(span_data.parent_id == 23); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == -1); + } else { + // W3C context was successfully extracted from traceparent header. + REQUIRE(span_data.trace_id == *test_case.expected_trace_id); + REQUIRE(span_data.parent_id == *test_case.expected_parent_id); + REQUIRE(decision); + REQUIRE(decision->origin == SamplingDecision::Origin::EXTRACTED); + REQUIRE(decision->priority == *test_case.expected_sampling_priority); + } + } + SECTION("x-datadog-tags") { auto finalized_config = finalize_config(config); REQUIRE(finalized_config); From b34231e8e6ba1a2237057b4c8147695b4c42310c Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 19:03:26 -0500 Subject: [PATCH 07/29] TODO: AMEND: add some documentation --- src/datadog/w3c_propagation.cpp | 4 +++- src/datadog/w3c_propagation.h | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 7b573460..a5f39dd1 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -13,7 +13,9 @@ namespace datadog { namespace tracing { namespace { -// TODO: document +// Populate the specified `result` with data extracted from the "traceparent" +// entry of the specified `headers`. Return `nullopt` on success. Return a value +// for the `tags::internal::w3c_extraction_error` tag if an error occurs. Optional extract_traceparent(ExtractedData& result, const DictReader& headers) { const auto maybe_traceparent = headers.lookup("traceparent"); diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 6be7d6f8..36486a1e 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -14,7 +14,11 @@ namespace tracing { class DictReader; -// TODO: document +// Return `ExtractedData` deduced from the "traceparent" and "tracestate" +// entries of the specified `headers`. If an error occurs, set a value for the +// `tags::internal::w3c_extraction_error` tag in the specified `span_tags`. +// `extract_w3c` will not return an error; instead, it returns an empty +// `ExtractedData` when extraction fails. Expected extract_w3c( const DictReader& headers, std::unordered_map& span_tags); From d0005cdec3b945db1cdf79892a11504868816890 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 20:26:56 -0500 Subject: [PATCH 08/29] TODO: AMEND: inject traceparent --- BUILD.bazel | 2 + CMakeLists.txt | 2 + src/datadog/hex.cpp | 1 + src/datadog/hex.h | 31 +++++++++++++ src/datadog/trace_segment.cpp | 26 +++-------- src/datadog/w3c_propagation.cpp | 30 +++++++++++++ src/datadog/w3c_propagation.h | 9 ++++ test/test_span.cpp | 79 +++++++++++++++++++++++++++++++++ 8 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 src/datadog/hex.cpp create mode 100644 src/datadog/hex.h diff --git a/BUILD.bazel b/BUILD.bazel index 290cb07e..d78fd98a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -18,6 +18,7 @@ cc_library( "src/datadog/expected.cpp", "src/datadog/extracted_data.cpp", "src/datadog/glob.cpp", + "src/datadog/hex.cpp", "src/datadog/http_client.cpp", "src/datadog/id_generator.cpp", "src/datadog/limiter.cpp", @@ -69,6 +70,7 @@ cc_library( "src/datadog/expected.h", "src/datadog/extracted_data.h", "src/datadog/glob.h", + "src/datadog/hex.h", "src/datadog/http_client.h", "src/datadog/id_generator.h", "src/datadog/json.hpp", diff --git a/CMakeLists.txt b/CMakeLists.txt index cb42dbe4..9c93261e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ target_sources(dd_trace_cpp PRIVATE src/datadog/expected.cpp src/datadog/extracted_data.cpp src/datadog/glob.cpp + src/datadog/hex.cpp src/datadog/http_client.cpp src/datadog/id_generator.cpp src/datadog/limiter.cpp @@ -129,6 +130,7 @@ target_sources(dd_trace_cpp PUBLIC src/datadog/expected.h src/datadog/extracted_data.h src/datadog/glob.h + src/datadog/hex.h src/datadog/http_client.h src/datadog/id_generator.h src/datadog/json_fwd.hpp diff --git a/src/datadog/hex.cpp b/src/datadog/hex.cpp new file mode 100644 index 00000000..fe81d19e --- /dev/null +++ b/src/datadog/hex.cpp @@ -0,0 +1 @@ +#include "hex.h" diff --git a/src/datadog/hex.h b/src/datadog/hex.h new file mode 100644 index 00000000..dec38a2e --- /dev/null +++ b/src/datadog/hex.h @@ -0,0 +1,31 @@ +#pragma once + +// This component provides a function, `hex`, for formatting an integral value +// in hexadecimal. + +#include +#include +#include +#include +#include + +namespace datadog { +namespace tracing { + +// Return the specified `value` formatted as a lower-case hexadecimal string +// without any leading zeroes. +template +std::string hex(Integer value) { + // 4 bits per hex digit char, and then +1 char for possible minus sign + char buffer[std::numeric_limits::digits / 4 + 1]; + + const int base = 16; + auto result = + std::to_chars(std::begin(buffer), std::end(buffer), value, base); + assert(result.ec == std::errc()); + + return std::string{std::begin(buffer), result.ptr}; +} + +} // namespace tracing +} // namespace datadog \ No newline at end of file diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 254fdba6..214960f0 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -1,10 +1,7 @@ #include "trace_segment.h" #include -#include -#include #include -#include #include #include @@ -12,6 +9,7 @@ #include "collector_response.h" #include "dict_writer.h" #include "error.h" +#include "hex.h" #include "logger.h" #include "optional.h" #include "span_data.h" @@ -19,25 +17,10 @@ #include "tag_propagation.h" #include "tags.h" #include "trace_sampler.h" +#include "w3c_propagation.h" namespace datadog { namespace tracing { -namespace { - -template -std::string hex(Integer value) { - // 4 bits per hex digit char, and then +1 char for possible minus sign - char buffer[std::numeric_limits::digits / 4 + 1]; - - const int base = 16; - auto result = - std::to_chars(std::begin(buffer), std::end(buffer), value, base); - assert(result.ec == std::errc()); - - return std::string{std::begin(buffer), result.ptr}; -} - -} // namespace TraceSegment::TraceSegment( const std::shared_ptr& logger, @@ -269,7 +252,10 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); break; case PropagationStyle::W3C: - // TODO + writer.set("traceparent", + encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, + span.span_id, sampling_priority)); + // TODO writer.set("tracestate", ...); break; default: assert(style == PropagationStyle::NONE); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index a5f39dd1..41e0113a 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -6,6 +6,7 @@ #include #include "dict_reader.h" +#include "hex.h" #include "parse_util.h" #include "tags.h" @@ -112,5 +113,34 @@ Expected extract_w3c( return result; } +std::string encode_traceparent( + std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, + std::uint64_t span_id, int sampling_priority) { + std::string result; + // version + result += "00-"; + + // trace ID + if (full_w3c_trace_id_hex) { + result += *full_w3c_trace_id_hex; + } else { + auto hexed = hex(trace_id); + result.append(16 + (16 - hexed.size()), '0'); // leading zeroes + result += hexed; + } + result += '-'; + + // span ID + auto hexed = hex(span_id); + result.append(16 - hexed.size(), '0'); // leading zeroes + result += hexed; + result += '-'; + + // flags + result += sampling_priority > 0 ? "01" : "00"; + + return result; +} + } // namespace tracing } // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 36486a1e..26e594dc 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -2,6 +2,7 @@ // TODO: document +#include #include #include @@ -23,5 +24,13 @@ Expected extract_w3c( const DictReader& headers, std::unordered_map& span_tags); +// Return a value for the "traceparent" header consisting of the specified +// `trace_id` or the optionally specified `full_w3c_trace_id_hex` as the trace +// ID, the specified `span_id` as the parent ID, and trace flags deduced from +// the specified `sampling_priority`. +std::string encode_traceparent( + std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, + std::uint64_t span_id, int sampling_priority); + } // namespace tracing } // namespace datadog diff --git a/test/test_span.cpp b/test/test_span.cpp index b976b2de..2acc13b3 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -3,6 +3,7 @@ // for propagation. #include +#include #include #include #include @@ -465,3 +466,81 @@ TEST_CASE("injection can be disabled using the \"none\" style") { const std::unordered_map empty; REQUIRE(writer.items == empty); } + +TEST_CASE("injecting W3C traceparent header") { + TracerConfig config; + config.defaults.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.injection_styles = {PropagationStyle::W3C}; + + SECTION("extracted from W3C traceparent") { + config.extraction_styles = {PropagationStyle::W3C}; + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + + // Override the tracer's ID generator to always return `expected_parent_id`. + constexpr std::uint64_t expected_parent_id = 0xcafebabe; + Tracer tracer{*finalized_config, [=]() { return expected_parent_id; }, + default_clock}; + + const std::unordered_map input_headers{ + // https://www.w3.org/TR/trace-context/#examples-of-http-traceparent-headers + {"traceparent", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + }; + const MockDictReader reader{input_headers}; + const auto maybe_span = tracer.extract_span(reader); + REQUIRE(maybe_span); + const auto& span = *maybe_span; + REQUIRE(span.id() == expected_parent_id); + + MockDictWriter writer; + span.inject(writer); + const auto& output_headers = writer.items; + const auto found = output_headers.find("traceparent"); + REQUIRE(found != output_headers.end()); + // The "00000000cafebabe" is the zero-padded `expected_parent_id`. + const StringView expected = + "00-4bf92f3577b34da6a3ce929d0e0e4736-00000000cafebabe-01"; + REQUIRE(found->second == expected); + } + + SECTION("not extracted from W3C traceparent") { + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + + // Override the tracer's ID generator to always return a fixed value. + // This will be both the trace ID and the parent (root) span ID. + constexpr std::uint64_t expected_id = 0xcafebabe; + Tracer tracer{*finalized_config, [=]() { return expected_id; }, + default_clock}; + + auto span = tracer.create_span(); + + // Let's test the effect sampling priority plays on the resulting + // traceparent, too. + struct TestCase { + int sampling_priority; + std::string expected_flags; + }; + const auto& [sampling_priority, expected_flags] = GENERATE( + values({{-1, "00"}, {0, "00"}, {1, "01"}, {2, "01"}})); + + CAPTURE(sampling_priority); + CAPTURE(expected_flags); + + span.trace_segment().override_sampling_priority(sampling_priority); + + MockDictWriter writer; + span.inject(writer); + const auto& output_headers = writer.items; + const auto found = output_headers.find("traceparent"); + REQUIRE(found != output_headers.end()); + // The "cafebabe"s come from `expected_id`. + const std::string expected = + "00-000000000000000000000000cafebabe-00000000cafebabe-" + + expected_flags; + REQUIRE(found->second == expected); + } +} From 5ce2166be3b9dd44ee9279f5eeed355c7988dede Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Tue, 20 Dec 2022 21:44:38 -0500 Subject: [PATCH 09/29] TODO: AMEND: untested tracestate extraction --- src/datadog/extracted_data.h | 3 +- src/datadog/tracer.cpp | 47 +++++---- src/datadog/w3c_propagation.cpp | 176 +++++++++++++++++++++++++++++++- src/datadog/w3c_propagation.h | 3 +- 4 files changed, 201 insertions(+), 28 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index 96fb468f..bfdd2dea 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -4,6 +4,7 @@ #include #include +#include #include "optional.h" @@ -14,7 +15,7 @@ struct ExtractedData { Optional trace_id; Optional parent_id; Optional origin; - Optional trace_tags; + std::unordered_map trace_tags; Optional sampling_priority; // TODO: document Optional full_w3c_trace_id_hex; diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 7dae2697..79d27dd3 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -26,6 +26,26 @@ namespace datadog { namespace tracing { namespace { +// Decode the specified `trace_tags` and integrate them into the specified +// `result`. If an error occurs, add a `tags::internal::propagation_error` tag +// to the specified `span_tags` and log a diagnostic using the specified +// `logger`. +void handle_trace_tags(StringView trace_tags, ExtractedData& result, + std::unordered_map& span_tags, + Logger& logger) { + auto maybe_trace_tags = decode_tags(trace_tags); + if (auto* error = maybe_trace_tags.if_error()) { + logger.log_error(*error); + span_tags[tags::internal::propagation_error] = "decoding_error"; + } else { + for (const auto& [key, value] : *maybe_trace_tags) { + if (starts_with(key, "_dd.p.")) { + result.trace_tags.insert_or_assign(key, value); + } + } + } +} + Expected> extract_id_header(const DictReader& headers, StringView header, StringView header_kind, @@ -54,7 +74,7 @@ Expected> extract_id_header(const DictReader& headers, Expected extract_datadog( const DictReader& headers, - std::unordered_map& /*span_tags*/) { + std::unordered_map& span_tags, Logger& logger) { ExtractedData result; auto trace_id = @@ -93,7 +113,7 @@ Expected extract_datadog( auto trace_tags = headers.lookup("x-datadog-tags"); if (trace_tags) { - result.trace_tags = std::string(*trace_tags); + handle_trace_tags(*trace_tags, result, span_tags, logger); } return result; @@ -101,7 +121,7 @@ Expected extract_datadog( Expected extract_b3( const DictReader& headers, - std::unordered_map& /*span_tags*/) { + std::unordered_map& span_tags, Logger& logger) { ExtractedData result; auto trace_id = extract_id_header(headers, "x-b3-traceid", "trace", "B3", 16); @@ -140,7 +160,7 @@ Expected extract_b3( auto trace_tags = headers.lookup("x-datadog-tags"); if (trace_tags) { - result.trace_tags = std::string(*trace_tags); + handle_trace_tags(*trace_tags, result, span_tags, logger); } return result; @@ -265,7 +285,7 @@ Expected Tracer::extract_span(const DictReader& reader, extracted_data = ExtractedData{}; continue; } - auto data = extract(reader, span_data->tags); + auto data = extract(reader, span_data->tags, *logger_); if (auto* error = data.if_error()) { return std::move(*error); } @@ -343,26 +363,11 @@ Expected Tracer::extract_span(const DictReader& reader, sampling_decision = decision; } - std::unordered_map decoded_trace_tags; - if (trace_tags) { - auto maybe_trace_tags = decode_tags(*trace_tags); - if (auto* error = maybe_trace_tags.if_error()) { - logger_->log_error(*error); - span_data->tags[tags::internal::propagation_error] = "decoding_error"; - } else { - for (const auto& [key, value] : *maybe_trace_tags) { - if (starts_with(key, "_dd.p.")) { - decoded_trace_tags.insert_or_assign(key, value); - } - } - } - } - const auto span_data_ptr = span_data.get(); const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, std::move(origin), tags_header_max_size_, - std::move(decoded_trace_tags), std::move(sampling_decision), + std::move(trace_tags), std::move(sampling_decision), std::move(full_w3c_trace_id_hex), std::move(additional_w3c_tracestate), std::move(additional_datadog_w3c_tracestate), std::move(span_data)); Span span{span_data_ptr, segment, generator_, clock_}; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 41e0113a..2e017e68 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -76,20 +76,186 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } +// TODO: document +struct PartiallyParsedTracestate { + StringView datadog_value; + std::string other_entries; +}; + +// TODO: document +Optional parse_tracestate(StringView tracestate) { + Optional result; + + const char* const begin = tracestate.begin(); + const char* const end = tracestate.end(); + const char* pair_begin = begin; + for (;;) { + const char* const pair_end = std::find(pair_begin, end, ','); + // Note that since this `pair` is `strip`ped, `pair_begin` is not + // necessarily equal to `pair.begin()` (similarly for the ends). + const auto pair = strip(range(pair_begin, pair_end)); + if (pair.empty()) { + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + const auto kv_separator = std::find(pair.begin(), pair.end(), '='); + if (kv_separator == pair.end()) { + // This is an invalid entry because it contains a non-whitespace character + // but not a "=". + // Let's move on to the next entry. + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + const auto key = range(pair.begin(), kv_separator); + if (key != "dd") { + // On to the next. + if (pair_end == end) { + return result; + } + pair_begin = pair_end + 1; + continue; + } + + // We found the "dd" entry. + result.emplace(); + result->datadog_value = range(kv_separator + 1, pair.end()); + // `result->other_entries` is whatever was before the "dd" entry and + // whatever is after the "dd" entry, but without a redundant comma in the + // middle. + if (pair_begin != begin) { + // There's a prefix + append(result->other_entries, range(begin, pair_begin - 1)); + if (pair_end != end) { + // and a suffix + append(result->other_entries, range(pair_end, end)); + } + } else if (pair_end != end) { + // There's just a suffix + append(result->other_entries, range(pair_end + 1, end)); + } + + return result; + } +} + +// TODO: document +Optional parse_datadog_tracestate(ExtractedData& result, + StringView datadog_value) { + const char* const begin = datadog_value.begin(); + const char* const end = datadog_value.end(); + const char* pair_begin = begin; + for (;;) { + const char* const pair_end = std::find(pair_begin, end, ';'); + const auto pair = range(pair_begin, pair_end); + if (pair.empty()) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + + const auto kv_separator = std::find(pair_begin, pair_end, ':'); + if (kv_separator == pair_end) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + + const auto key = range(pair_begin, kv_separator); + const auto value = range(kv_separator + 1, pair_end); + if (key == "o") { + result.origin = std::string{value}; + } else if (key == "s") { + const auto maybe_priority = parse_int(value, 10); + if (!maybe_priority) { + // chaff! + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + continue; + } + const int priority = *maybe_priority; + // If we didn't parse a sampling priority from traceparent, or if the one + // we just parsed from tracestate is consistent with the previous, then + // set the sampling priority to the one we just parsed. + // Alternatively, if we already parsed a sampling priority from + // traceparent and got a result inconsistent with that parsed here, go + // with the one previously parsed from traceparent. + if (!result.sampling_priority || + (*result.sampling_priority > 0) == (priority > 0)) { + result.sampling_priority = priority; + } + } else if (starts_with(key, "t.")) { + // The part of the key that follows "t." is the name of a trace tag, + // except without the "_dd.p." prefix. + const auto tag_suffix = key.substr(2); + std::string tag_name = "_dd.p."; + append(tag_name, tag_suffix); + result.trace_tags.insert_or_assign(std::move(tag_name), + std::string{value}); + } else { + // Unrecognized key: append the whole pair to + // `additional_datadog_w3c_tracestate`, which will be used if/when we + // inject trace context. + auto& entries = result.additional_datadog_w3c_tracestate; + if (!entries) { + entries.emplace(); + } + if (!entries->empty()) { + *entries += ';'; + } + append(*entries, pair); + } + + if (pair_end == end) { + break; + } + pair_begin = pair_end + 1; + } + + return nullopt; +} + // TODO: document Optional extract_tracestate(ExtractedData& result, const DictReader& headers) { - // TODO - (void)result; - (void)headers; - return nullopt; + const auto maybe_tracestate = headers.lookup("tracestate"); + if (!maybe_tracestate) { + return nullopt; + } + + const auto tracestate = strip(*maybe_tracestate); + auto maybe_parsed = parse_tracestate(tracestate); + if (!maybe_parsed) { + // No "dd" entry in `tracestate`, so there's nothing to extract. + result.additional_w3c_tracestate = std::string{tracestate}; + return nullopt; + } + + auto& [datadog_value, other_entries] = *maybe_parsed; + result.additional_datadog_w3c_tracestate = std::move(other_entries); + return parse_datadog_tracestate(result, datadog_value); } } // namespace Expected extract_w3c( const DictReader& headers, - std::unordered_map& span_tags) { + std::unordered_map& span_tags, Logger&) { ExtractedData result; if (auto error_tag_value = extract_traceparent(result, headers)) { diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 26e594dc..3d89ad02 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -14,6 +14,7 @@ namespace datadog { namespace tracing { class DictReader; +class Logger; // Return `ExtractedData` deduced from the "traceparent" and "tracestate" // entries of the specified `headers`. If an error occurs, set a value for the @@ -22,7 +23,7 @@ class DictReader; // `ExtractedData` when extraction fails. Expected extract_w3c( const DictReader& headers, - std::unordered_map& span_tags); + std::unordered_map& span_tags, Logger&); // Return a value for the "traceparent" header consisting of the specified // `trace_id` or the optionally specified `full_w3c_trace_id_hex` as the trace From 3158907ec24c948cd1978ca7b65e95ee91c788f3 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 11:48:38 -0500 Subject: [PATCH 10/29] TODO: AMEND: tested tracestate extraction --- src/datadog/w3c_propagation.cpp | 86 ++++++-------- test/mocks/loggers.cpp | 16 +++ test/mocks/loggers.h | 3 + test/test.cpp | 8 ++ test/test.h | 2 + test/test_tracer.cpp | 203 ++++++++++++++++++++++++++++++++ 6 files changed, 266 insertions(+), 52 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 2e017e68..45a49633 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -1,5 +1,6 @@ #include "w3c_propagation.h" +#include #include #include #include @@ -27,7 +28,7 @@ Optional extract_traceparent(ExtractedData& result, const auto traceparent = strip(*maybe_traceparent); // Note that leading and trailing whitespace was already removed above. - // Note that the match group 0 is the entire match. + // Note that match group 0 is the entire match. static const auto& pattern = "([0-9a-f]{2})" // hex version number (match group 1) "-" @@ -89,16 +90,13 @@ Optional parse_tracestate(StringView tracestate) { const char* const begin = tracestate.begin(); const char* const end = tracestate.end(); const char* pair_begin = begin; - for (;;) { + while (pair_begin != end) { const char* const pair_end = std::find(pair_begin, end, ','); // Note that since this `pair` is `strip`ped, `pair_begin` is not // necessarily equal to `pair.begin()` (similarly for the ends). const auto pair = strip(range(pair_begin, pair_end)); if (pair.empty()) { - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -107,20 +105,14 @@ Optional parse_tracestate(StringView tracestate) { // This is an invalid entry because it contains a non-whitespace character // but not a "=". // Let's move on to the next entry. - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const auto key = range(pair.begin(), kv_separator); if (key != "dd") { // On to the next. - if (pair_end == end) { - return result; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -128,7 +120,7 @@ Optional parse_tracestate(StringView tracestate) { result.emplace(); result->datadog_value = range(kv_separator + 1, pair.end()); // `result->other_entries` is whatever was before the "dd" entry and - // whatever is after the "dd" entry, but without a redundant comma in the + // whatever is after the "dd" entry, but without an extra comma in the // middle. if (pair_begin != begin) { // There's a prefix @@ -142,35 +134,30 @@ Optional parse_tracestate(StringView tracestate) { append(result->other_entries, range(pair_end + 1, end)); } - return result; + break; } + + return result; } // TODO: document -Optional parse_datadog_tracestate(ExtractedData& result, - StringView datadog_value) { +void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { const char* const begin = datadog_value.begin(); const char* const end = datadog_value.end(); const char* pair_begin = begin; - for (;;) { + while (pair_begin != end) { const char* const pair_end = std::find(pair_begin, end, ';'); const auto pair = range(pair_begin, pair_end); if (pair.empty()) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const auto kv_separator = std::find(pair_begin, pair_end, ':'); if (kv_separator == pair_end) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } @@ -182,10 +169,7 @@ Optional parse_datadog_tracestate(ExtractedData& result, const auto maybe_priority = parse_int(value, 10); if (!maybe_priority) { // chaff! - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; continue; } const int priority = *maybe_priority; @@ -205,8 +189,12 @@ Optional parse_datadog_tracestate(ExtractedData& result, const auto tag_suffix = key.substr(2); std::string tag_name = "_dd.p."; append(tag_name, tag_suffix); + // The tag value was encoded with all '=' replaced by '~'. Undo that + // transformation. + std::string decoded_value{value}; + std::replace(decoded_value.begin(), decoded_value.end(), '~', '='); result.trace_tags.insert_or_assign(std::move(tag_name), - std::string{value}); + std::move(decoded_value)); } else { // Unrecognized key: append the whole pair to // `additional_datadog_w3c_tracestate`, which will be used if/when we @@ -214,41 +202,39 @@ Optional parse_datadog_tracestate(ExtractedData& result, auto& entries = result.additional_datadog_w3c_tracestate; if (!entries) { entries.emplace(); - } - if (!entries->empty()) { + } else { *entries += ';'; } append(*entries, pair); } - if (pair_end == end) { - break; - } - pair_begin = pair_end + 1; + pair_begin = pair_end == end ? end : pair_end + 1; } - - return nullopt; } // TODO: document -Optional extract_tracestate(ExtractedData& result, - const DictReader& headers) { +void extract_tracestate(ExtractedData& result, const DictReader& headers) { const auto maybe_tracestate = headers.lookup("tracestate"); if (!maybe_tracestate) { - return nullopt; + return; } const auto tracestate = strip(*maybe_tracestate); auto maybe_parsed = parse_tracestate(tracestate); if (!maybe_parsed) { // No "dd" entry in `tracestate`, so there's nothing to extract. - result.additional_w3c_tracestate = std::string{tracestate}; - return nullopt; + if (!tracestate.empty()) { + result.additional_w3c_tracestate = std::string{tracestate}; + } + return; } auto& [datadog_value, other_entries] = *maybe_parsed; - result.additional_datadog_w3c_tracestate = std::move(other_entries); - return parse_datadog_tracestate(result, datadog_value); + if (!other_entries.empty()) { + result.additional_w3c_tracestate = std::move(other_entries); + } + + parse_datadog_tracestate(result, datadog_value); } } // namespace @@ -270,11 +256,7 @@ Expected extract_w3c( return result; } - if (auto error_tag_value = extract_tracestate(result, headers)) { - span_tags[tags::internal::w3c_extraction_error] = - std::move(*error_tag_value); - // Carry on with whatever data was extracted from traceparent. - } + extract_tracestate(result, headers); return result; } diff --git a/test/mocks/loggers.cpp b/test/mocks/loggers.cpp index a7655dfd..795a8a75 100644 --- a/test/mocks/loggers.cpp +++ b/test/mocks/loggers.cpp @@ -1 +1,17 @@ #include "loggers.h" + +#include +#include + +std::ostream& operator<<(std::ostream& stream, + const std::vector& entries) { + stream << ""; + for (std::size_t i = 0; i < entries.size(); ++i) { + const auto& entry = entries[i]; + const auto kind_name = + entry.kind == MockLogger::Entry::ERROR ? "ERROR" : "STARTUP"; + stream << '\n' << (i + 1) << ". " << kind_name << ": "; + std::visit([&](const auto& value) { stream << value; }, entry.payload); + } + return stream << ""; +} diff --git a/test/mocks/loggers.h b/test/mocks/loggers.h index 5364a1c4..c41b4278 100644 --- a/test/mocks/loggers.h +++ b/test/mocks/loggers.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -97,3 +98,5 @@ struct MockLogger : public Logger { return std::get(found->payload); } }; + +std::ostream& operator<<(std::ostream&, const std::vector&); diff --git a/test/test.cpp b/test/test.cpp index ca57ed79..801d68a7 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -14,4 +14,12 @@ std::ostream& operator<<( return stream << item.value_or(""); } +std::ostream& operator<<(std::ostream& stream, + const std::optional& maybe) { + if (maybe) { + return stream << *maybe; + } + return stream << ""; +} + } // namespace std diff --git a/test/test.h b/test/test.h index e9a8664a..0cf17b1b 100644 --- a/test/test.h +++ b/test/test.h @@ -26,6 +26,8 @@ std::ostream& operator<<(std::ostream& stream, std::ostream& operator<<(std::ostream& stream, const std::optional& maybe); +std::ostream& operator<<(std::ostream& stream, const std::optional& maybe); + } // namespace std namespace datadog { diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 981c2b35..7142c17d 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -651,6 +652,208 @@ TEST_CASE("span extraction") { } } + SECTION("W3C tracestate extraction") { + // Ideally this would test the _behavior_ of W3C tracestate extraction, + // rather than its implementation. + // However, some of the effects of W3C tracestate extraction cannot be + // observed except by injecting trace context, and there's a separate test + // for W3C tracestate injection (in `test_span.cpp`). + // Here we test the tracestate portion of the `extract_w3c` function, + // declared in `w3c_propagation.h`. + struct TestCase { + int line; + std::string name; + std::string traceparent; + Optional tracestate; + Optional expected_sampling_priority = {}; + Optional expected_origin = {}; + std::unordered_map expected_trace_tags = {}; + Optional expected_additional_w3c_tracestate = {}; + Optional expected_additional_datadog_w3c_tracestate = {}; + }; + + static const std::string traceparent_prefix = + "00-00000000000000000000000000000001-0000000000000001-0"; + static const std::string traceparent_drop = traceparent_prefix + "0"; + static const std::string traceparent_keep = traceparent_prefix + "1"; + // clang-format off + auto test_case = GENERATE(values({ + {__LINE__, "no tracestate", + traceparent_drop, // traceparent + nullopt, // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "empty tracestate", + traceparent_drop, // traceparent + "", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "no dd entry", + traceparent_drop, // traceparent + "foo=hello,@thingy/thing=wah;wah;wah", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,@thingy/thing=wah;wah;wah", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "empty entry", + traceparent_drop, // traceparent + "foo=hello,,bar=thing", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,,bar=thing", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "malformed entry", + traceparent_drop, // traceparent + "foo=hello,chicken,bar=thing", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,chicken,bar=thing", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff before dd entry", + traceparent_drop, // traceparent + "foo=hello,bar=baz,dd=", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff after dd entry", + traceparent_drop, // traceparent + "dd=,foo=hello,bar=baz", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "stuff before and after dd entry", + traceparent_drop, // traceparent + "chicken=yes,nuggets=yes,dd=,foo=hello,bar=baz", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + "chicken=yes,nuggets=yes,foo=hello,bar=baz", // expected_additional_w3c_tracestate + nullopt}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "dd entry with empty subentries", + traceparent_drop, // traceparent + "dd=foo:bar;;;;;baz:bam;;;", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "foo:bar;baz:bam"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "dd entry with malformed subentries", + traceparent_drop, // traceparent + "dd=foo:bar;chicken;chicken;baz:bam;chicken", // tracestate + 0, // expected_sampling_priority + nullopt, // expected_origin + {}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "foo:bar;baz:bam"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "origin, trace tags, and extra fields", + traceparent_drop, // traceparent + "dd=o:France;t.foo:thing1;t.bar:thing2;x:wow;y:wow", // tracestate + 0, // expected_sampling_priority + "France", // expected_origin + {{"_dd.p.foo", "thing1"}, {"_dd.p.bar", "thing2"}}, // expected_trace_tags + nullopt, // expected_additional_w3c_tracestate + "x:wow;y:wow"}, // expected_additional_datadog_w3c_tracestate + + {__LINE__, "traceparent and tracestate sampling agree (1/4)", + traceparent_drop, // traceparent + "dd=s:0", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (2/4)", + traceparent_drop, // traceparent + "dd=s:-1", // tracestate + -1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (3/4)", + traceparent_keep, // traceparent + "dd=s:1", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling agree (4/4)", + traceparent_keep, // traceparent + "dd=s:2", // tracestate + 2}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (1/4)", + traceparent_drop, // traceparent + "dd=s:1", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (2/4)", + traceparent_drop, // traceparent + "dd=s:2", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (3/4)", + traceparent_keep, // traceparent + "dd=s:0", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "traceparent and tracestate sampling disagree (4/4)", + traceparent_keep, // traceparent + "dd=s:-1", // tracestate + 1}, // expected_sampling_priority + + {__LINE__, "invalid sampling priority (1/2)", + traceparent_drop, // traceparent + "dd=s:oops", // tracestate + 0}, // expected_sampling_priority + + {__LINE__, "invalid sampling priority (2/2)", + traceparent_keep, // traceparent + "dd=s:oops", // tracestate + 1}, // expected_sampling_priority + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + CAPTURE(test_case.traceparent); + CAPTURE(test_case.tracestate); + + std::unordered_map span_tags; + MockLogger logger; + CAPTURE(logger.entries); + CAPTURE(span_tags); + + std::unordered_map headers; + headers["traceparent"] = test_case.traceparent; + if (test_case.tracestate) { + headers["tracestate"] = *test_case.tracestate; + } + MockDictReader reader{headers}; + + const auto extracted = extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + + REQUIRE(extracted->origin == test_case.expected_origin); + REQUIRE(extracted->trace_tags == test_case.expected_trace_tags); + REQUIRE(extracted->sampling_priority == + test_case.expected_sampling_priority); + REQUIRE(extracted->additional_w3c_tracestate == + test_case.expected_additional_w3c_tracestate); + REQUIRE(extracted->additional_datadog_w3c_tracestate == + test_case.expected_additional_datadog_w3c_tracestate); + + REQUIRE(logger.entries.empty()); + REQUIRE(span_tags.empty()); + } + SECTION("x-datadog-tags") { auto finalized_config = finalize_config(config); REQUIRE(finalized_config); From b69ad49b82bcb630a8d0075d3977d1624929cc2d Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 15:29:08 -0500 Subject: [PATCH 11/29] TODO: AMEND: add some more documentation --- src/datadog/extracted_data.h | 22 ++++++++++++++++++---- src/datadog/w3c_propagation.cpp | 24 ++++++++++++++++++++---- src/datadog/w3c_propagation.h | 4 +++- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index bfdd2dea..f6d90f82 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -1,6 +1,7 @@ #pragma once -// TODO: document +// This component provides a `struct`, `ExtractedData`, that stores fields +// extracted from trace context. It's an implementation detail of this library. #include #include @@ -17,11 +18,24 @@ struct ExtractedData { Optional origin; std::unordered_map trace_tags; Optional sampling_priority; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C` + // and trace context was successfully extracted, then `full_w3c_trace_id_hex` + // contains the hex-encoded 128-bit trace ID. `trace_id` will be the least + // significant 64 bits of the same value. `full_w3c_trace_id_hex` is used for + // the `W3C` injection style. Optional full_w3c_trace_id_hex; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, + // then `additional_w3c_tracestate` contains the parts of the "tracestate" + // header that are not the "dd" (Datadog) entry. If there are no other parts, + // then `additional_w3c_tracestate` is null. + // `additional_w3c_tracestate` is used for the `W3C` injection style. Optional additional_w3c_tracestate; - // TODO: document + // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, + // and if the "tracestate" header contained a "dd" (Datadog) entry, then + // `additional_datadog_w3c_tracestate` contains fields from within the "dd" + // entry that were not interpreted. If there are no such fields, then + // `additional_datadog_w3c_tracestate` is null. + // `additional_datadog_w3c_tracestate` is used for the `W3C` injection style. Optional additional_datadog_w3c_tracestate; }; diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 45a49633..ec69d4c5 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -77,13 +77,16 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } -// TODO: document +// `struct PartiallyParsedTracestat` contains the separated Datadog-specific and +// non-Datadog-specific portions of tracestate. struct PartiallyParsedTracestate { StringView datadog_value; std::string other_entries; }; -// TODO: document +// Return the separate Datadog-specific and non-Datadog-specific portions of the +// specified `tracestate`. If `tracestate` does not have a Datadog-specific +// portion, return `nullopt`. Optional parse_tracestate(StringView tracestate) { Optional result; @@ -140,7 +143,15 @@ Optional parse_tracestate(StringView tracestate) { return result; } -// TODO: document +// Fill the specified `result` with information parsed from the specified +// `datadog_value`. `datadog_value` is the value of the "dd" entry in the +// "tracestate" header. +// +// `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// - `origin` +// - `trace_tags` +// - `sampling_priority` +// - `additional_datadog_w3c_tracestate` void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { const char* const begin = datadog_value.begin(); const char* const end = datadog_value.end(); @@ -212,7 +223,12 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { } } -// TODO: document +// Fill the specified `result` with information parsed from the "tracestate" +// element of the specified `headers`, if present. +// +// `extract_tracestate` populates the `additional_w3c_tracestate` field of +// `ExtractedData`, in addition to those populated by +// `parse_datadog_tracestate`. void extract_tracestate(ExtractedData& result, const DictReader& headers) { const auto maybe_tracestate = headers.lookup("tracestate"); if (!maybe_tracestate) { diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 3d89ad02..16096a5b 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -1,6 +1,8 @@ #pragma once -// TODO: document +// This component provides functions for extracting and injecting trace context +// in the `PropagationStyle::W3C` style. These functions decode and encode the +// "traceparent" and "tracestate" HTTP request headers. #include #include From fea6cae4f017f72d4198f63acaa91b9605bdb873 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 16:12:59 -0500 Subject: [PATCH 12/29] TODO: AMEND: checkpoint --- src/datadog/trace_segment.cpp | 89 ++++++++++++++++++++++----------- src/datadog/w3c_propagation.cpp | 37 ++++++++++++++ src/datadog/w3c_propagation.h | 7 +++ 3 files changed, 103 insertions(+), 30 deletions(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 214960f0..0b67ab4d 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -21,6 +21,38 @@ namespace datadog { namespace tracing { +namespace { + +// Encode the specified `trace_tags`. If the encoded value is not longer than +// the specified `tags_header_max_size`, then set it as the "x-datadog-tags" +// header using the specified `writer`. If the encoded value is oversized, then +// write a diagnostic to the specified `logger` and set a propagation error tag +// on the specified `local_root_tags`. +void inject_trace_tags( + DictWriter& writer, + const std::unordered_map& trace_tags, + std::size_t tags_header_max_size, + std::unordered_map& local_root_tags, + Logger& logger) { + const std::string encoded_trace_tags = encode_tags(trace_tags); + + if (encoded_trace_tags.size() > tags_header_max_size) { + std::string message; + message += + "Serialized x-datadog-tags header value is too large. The configured " + "maximum size is "; + message += std::to_string(tags_header_max_size); + message += " bytes, but the encoded value is "; + message += std::to_string(encoded_trace_tags.size()); + message += " bytes."; + logger.log_error(message); + local_root_tags[tags::internal::propagation_error] = "inject_max_size"; + } else if (!encoded_trace_tags.empty()) { + writer.set("x-datadog-tags", encoded_trace_tags); + } +} + +} // namespace TraceSegment::TraceSegment( const std::shared_ptr& logger, @@ -201,41 +233,24 @@ void TraceSegment::update_decision_maker_trace_tag() { } void TraceSegment::inject(DictWriter& writer, const SpanData& span) { + // If the only injection style is `NONE`, then don't do anything. + if (injection_styles_.size() == 1 && + injection_styles_[0] == PropagationStyle::NONE) { + return; + } + + // The sampling priority can change (it can be overridden on another thread), + // and trace tags might change when that happens ("_dd.p.dm"). + // So, we lock here, make a sampling decision if necessary, and then copy the + // decision and trace tags before unlocking. int sampling_priority; - std::string encoded_trace_tags; + std::unordered_map trace_tags; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); assert(sampling_decision_); sampling_priority = sampling_decision_->priority; - encoded_trace_tags = encode_tags(trace_tags_); - } - - // Origin and trace tag headers are always propagated, unless the only - // injection style is "none". - // Other headers depend on the injection styles. - if (injection_styles_.size() == 1 && - injection_styles_[0] == PropagationStyle::NONE) { - return; - } - - if (origin_) { - writer.set("x-datadog-origin", *origin_); - } - if (encoded_trace_tags.size() > tags_header_max_size_) { - std::string message; - message += - "Serialized x-datadog-tags header value is too large. The configured " - "maximum size is "; - message += std::to_string(tags_header_max_size_); - message += " bytes, but the encoded value is "; - message += std::to_string(encoded_trace_tags.size()); - message += " bytes."; - logger_->log_error(message); - SpanData& local_root = *spans_.front(); - local_root.tags[tags::internal::propagation_error] = "inject_max_size"; - } else if (!encoded_trace_tags.empty()) { - writer.set("x-datadog-tags", encoded_trace_tags); + trace_tags = trace_tags_; } for (const auto style : injection_styles_) { @@ -245,17 +260,31 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", std::to_string(sampling_priority)); + if (origin_) { + writer.set("x-datadog-origin", *origin_); + } + inject_trace_tags(writer, trace_tags, tags_header_max_size_, + spans_.front()->tags, *logger_); break; case PropagationStyle::B3: writer.set("x-b3-traceid", hex(span.trace_id)); writer.set("x-b3-spanid", hex(span.span_id)); writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); + if (origin_) { + writer.set("x-datadog-origin", *origin_); + } + inject_trace_tags(writer, trace_tags, tags_header_max_size_, + spans_.front()->tags, *logger_); break; case PropagationStyle::W3C: writer.set("traceparent", encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, span.span_id, sampling_priority)); - // TODO writer.set("tracestate", ...); + // TODO: handle oversized things. + writer.set("tracestate", + encode_tracestate(sampling_priority, origin_, trace_tags, + additional_datadog_w3c_tracestate_, + additional_w3c_tracestate_)); break; default: assert(style == PropagationStyle::NONE); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index ec69d4c5..c7b54f57 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -306,5 +306,42 @@ std::string encode_traceparent( return result; } +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate) { + std::string result = "dd=s:"; + result += std::to_string(sampling_priority); + + if (origin) { + result += ";o:"; + result += *origin; + } + + for (const auto& [key, value] : trace_tags) { + // `key` is "_dd.p.", but we want "t.". + result += ";t."; + result.append(key, sizeof("_dd.p.") - 1); + + // `value` might contain equal signs ("="), which is reserved in tracestate. + // Replace them with tildes ("~"). + result += value; + std::replace(result.end() - value.size(), result.end(), '=', '~'); + } + + if (additional_datadog_w3c_tracestate) { + result += ';'; + result += *additional_datadog_w3c_tracestate; + } + + if (additional_w3c_tracestate) { + result += ','; + result += *additional_w3c_tracestate; + } + + return result; +} + } // namespace tracing } // namespace datadog diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 16096a5b..fe11724e 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -35,5 +35,12 @@ std::string encode_traceparent( std::uint64_t trace_id, const Optional& full_w3c_trace_id_hex, std::uint64_t span_id, int sampling_priority); +// Return a value for the "tracestate" header containing the specified fields. +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate); + } // namespace tracing } // namespace datadog From b1888ec165af07b92924a6b4d6d62fb7e642cd20 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 17:12:37 -0500 Subject: [PATCH 13/29] TODO: AMEND: checkpoint --- src/datadog/trace_segment.cpp | 1 - src/datadog/w3c_propagation.cpp | 63 ++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 0b67ab4d..8b2ec50e 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -280,7 +280,6 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { writer.set("traceparent", encode_traceparent(span.trace_id, full_w3c_trace_id_hex_, span.span_id, sampling_priority)); - // TODO: handle oversized things. writer.set("tracestate", encode_tracestate(sampling_priority, origin_, trace_tags, additional_datadog_w3c_tracestate_, diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index c7b54f57..526f258a 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -15,6 +15,22 @@ namespace datadog { namespace tracing { namespace { +// Return a predicate that returns whether its `char` argument is any of the +// following: +// +// - outside of the ASCII inclusive range `[lowest_ascii, highest_ascii]` +// - equal to one of the `disallowed_characters`. +// +// `verboten` is used as an argument to `std::replace_if` to sanitize field +// values within the tracestate header. +auto verboten(int lowest_ascii, int highest_ascii, + StringView disallowed_characters) { + return [=, chars = disallowed_characters](char ch) { + return int(ch) < lowest_ascii || int(ch) > highest_ascii || + std::find(chars.begin(), chars.end(), ch) != chars.end(); + }; +} + // Populate the specified `result` with data extracted from the "traceparent" // entry of the specified `headers`. Return `nullopt` on success. Return a value // for the `tags::internal::w3c_extraction_error` tag if an error occurs. @@ -148,6 +164,7 @@ Optional parse_tracestate(StringView tracestate) { // "tracestate" header. // // `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// // - `origin` // - `trace_tags` // - `sampling_priority` @@ -306,28 +323,49 @@ std::string encode_traceparent( return result; } -std::string encode_tracestate( +std::string encode_datadog_tracestate( int sampling_priority, const Optional& origin, const std::unordered_map& trace_tags, - const Optional& additional_datadog_w3c_tracestate, - const Optional& additional_w3c_tracestate) { + const Optional& additional_datadog_w3c_tracestate) { std::string result = "dd=s:"; result += std::to_string(sampling_priority); + const std::size_t max_size = 256; + std::size_t last_good_size = result.size(); + if (origin) { result += ";o:"; result += *origin; + std::replace_if(result.end() - origin->size(), result.end(), + verboten(0x20, 0x7e, ",;="), '_'); } + if (result.size() > max_size) { + result.resize(last_good_size); + return result; + } + last_good_size = result.size(); + for (const auto& [key, value] : trace_tags) { // `key` is "_dd.p.", but we want "t.". result += ";t."; - result.append(key, sizeof("_dd.p.") - 1); + const auto prefix_length = sizeof("_dd.p.") - 1; + result.append(key, prefix_length); + std::replace_if(result.end() - (key.size() - prefix_length), result.end(), + verboten(0x20, 0x7e, " ,;="), '_'); + result += value; + std::replace_if(result.end() - value.size(), result.end(), + verboten(0x20, 0x7e, ",;~"), '_'); // `value` might contain equal signs ("="), which is reserved in tracestate. // Replace them with tildes ("~"). - result += value; std::replace(result.end() - value.size(), result.end(), '=', '~'); + + if (result.size() > max_size) { + result.resize(last_good_size); + return result; + } + last_good_size = result.size(); } if (additional_datadog_w3c_tracestate) { @@ -335,6 +373,21 @@ std::string encode_tracestate( result += *additional_datadog_w3c_tracestate; } + if (result.size() > max_size) { + result.resize(last_good_size); + } + + return result; +} + +std::string encode_tracestate( + int sampling_priority, const Optional& origin, + const std::unordered_map& trace_tags, + const Optional& additional_datadog_w3c_tracestate, + const Optional& additional_w3c_tracestate) { + std::string result = encode_datadog_tracestate( + sampling_priority, origin, trace_tags, additional_datadog_w3c_tracestate); + if (additional_w3c_tracestate) { result += ','; result += *additional_w3c_tracestate; From 6bc31683e867b19437f94288f1d2644d8ea6392f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 19:46:00 -0500 Subject: [PATCH 14/29] TODO: AMEND: checkpoint, before I mess with flat maps --- src/datadog/tags.cpp | 1 - src/datadog/tags.h | 1 - src/datadog/w3c_propagation.cpp | 17 ++++--- test/mocks/loggers.cpp | 2 +- test/test_span.cpp | 85 +++++++++++++++++++++++++++++++++ test/test_tracer_config.cpp | 18 +++---- 6 files changed, 106 insertions(+), 18 deletions(-) diff --git a/src/datadog/tags.cpp b/src/datadog/tags.cpp index 84b91202..db5828b2 100644 --- a/src/datadog/tags.cpp +++ b/src/datadog/tags.cpp @@ -29,7 +29,6 @@ const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism"; const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate"; const std::string span_sampling_limit = "_dd.span_sampling.max_per_second"; const std::string w3c_extraction_error = "_dd.w3c_extraction_error"; -const std::string w3c_injection_error = "_dd.w3c_injection_error"; } // namespace internal diff --git a/src/datadog/tags.h b/src/datadog/tags.h index b1d77c58..996cc58b 100644 --- a/src/datadog/tags.h +++ b/src/datadog/tags.h @@ -33,7 +33,6 @@ extern const std::string span_sampling_mechanism; extern const std::string span_sampling_rule_rate; extern const std::string span_sampling_limit; extern const std::string w3c_extraction_error; -extern const std::string w3c_injection_error; } // namespace internal // Return whether the specified `tag_name` is reserved for use internal to this diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 526f258a..87935829 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -21,8 +21,8 @@ namespace { // - outside of the ASCII inclusive range `[lowest_ascii, highest_ascii]` // - equal to one of the `disallowed_characters`. // -// `verboten` is used as an argument to `std::replace_if` to sanitize field -// values within the tracestate header. +// `verboten` is used with `std::replace_if` to sanitize field values within the +// tracestate header. auto verboten(int lowest_ascii, int highest_ascii, StringView disallowed_characters) { return [=, chars = disallowed_characters](char ch) { @@ -347,13 +347,18 @@ std::string encode_datadog_tracestate( last_good_size = result.size(); for (const auto& [key, value] : trace_tags) { - // `key` is "_dd.p.", but we want "t.". + const StringView prefix = "_dd.p."; + if (!starts_with(key, prefix)) { + continue; + } + + // `key` is "_dd.p.", but we want "t.". result += ";t."; - const auto prefix_length = sizeof("_dd.p.") - 1; - result.append(key, prefix_length); - std::replace_if(result.end() - (key.size() - prefix_length), result.end(), + result.append(key, prefix.size()); + std::replace_if(result.end() - (key.size() - prefix.size()), result.end(), verboten(0x20, 0x7e, " ,;="), '_'); + result += ':'; result += value; std::replace_if(result.end() - value.size(), result.end(), verboten(0x20, 0x7e, ",;~"), '_'); diff --git a/test/mocks/loggers.cpp b/test/mocks/loggers.cpp index 795a8a75..b7c7e962 100644 --- a/test/mocks/loggers.cpp +++ b/test/mocks/loggers.cpp @@ -13,5 +13,5 @@ std::ostream& operator<<(std::ostream& stream, stream << '\n' << (i + 1) << ". " << kind_name << ": "; std::visit([&](const auto& value) { stream << value; }, entry.payload); } - return stream << ""; + return stream << "\n"; } diff --git a/test/test_span.cpp b/test/test_span.cpp index 2acc13b3..be331201 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -544,3 +544,88 @@ TEST_CASE("injecting W3C traceparent header") { REQUIRE(found->second == expected); } } + +TEST_CASE("injecting W3C tracestate header") { + // Concerns: + // - the basics: + // - sampling priority + // - origin + // - trace tags + // - extra fields (extracted from W3C) + // - all of the above + // - character substitutions: + // - in origin + // - in trace tag key + // - in trace tag value + // - special tilde ("~") behavior + // - length limit: + // - at origin + // - at a trace tag (extract extra fields from W3C) + // - at the extra fields (extracted from W3C) + + TracerConfig config; + config.defaults.service = "testsvc"; + // The order of the extraction styles doesn't matter for this test, because + // it'll either be one or the other in the test cases. + config.extraction_styles = {PropagationStyle::DATADOG, PropagationStyle::W3C}; + config.injection_styles = {PropagationStyle::W3C}; + // If one of these test cases results in a local sampling decision, let it be + // "drop." + config.trace_sampler.sample_rate = 0.0; + const auto logger = std::make_shared(); + config.logger = logger; + config.collector = std::make_shared(); + + const auto finalized_config = finalize_config(config); + REQUIRE(finalized_config); + Tracer tracer{*finalized_config}; + + struct TestCase { + int line; + std::string name; + std::unordered_map input_headers; + std::string expected_tracestate; + }; + + // clang-format off + auto test_case = GENERATE(values({ + {__LINE__, "sampling priority", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-sampling-priority", "2"}}, + "dd=s:2"}, + + {__LINE__, "origin", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "France"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;o:France"}, + + {__LINE__, "trace tags", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.foo:x;t.bar:y"}, + })); + // clang-format on + + CAPTURE(test_case.name); + CAPTURE(test_case.line); + CAPTURE(test_case.input_headers); + CAPTURE(test_case.expected_tracestate); + CAPTURE(logger->entries); + + MockDictReader reader{test_case.input_headers}; + const auto span = tracer.extract_span(reader); + REQUIRE(span); + + MockDictWriter writer; + span->inject(writer); + + CAPTURE(writer.items); + const auto found = writer.items.find("tracestate"); + REQUIRE(found != writer.items.end()); + + REQUIRE(found->second == test_case.expected_tracestate); + + REQUIRE(logger->error_count() == 0); +} diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index 27e4bb85..a106875f 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -1158,19 +1158,19 @@ TEST_CASE("TracerConfig propagation styles") { const char* const vars[] = {ts, tse, se, tsi, si}; constexpr auto n = sizeof(vars) / sizeof(vars[0]); // clang-format off - const bool x = false; // ignored values - const bool expect_warning[n][n] = { - // ts tse se tsi si - // --- --- --- --- --- - /* ts */{ x, true, true, true, true }, + const bool x = false; // ignored values + const bool expect_warning[n][n] = { + // ts tse se tsi si + // --- --- --- --- --- + /* ts */{ x, true, true, true, true }, - /* tse */{ x, x, true, false, false }, + /* tse */{ x, x, true, false, false }, - /* se */{ x, x, x, false, false }, + /* se */{ x, x, x, false, false }, - /* tsi */{ x, x, x, x, true }, + /* tsi */{ x, x, x, x, true }, - /* si */{ x, x, x, x, x }, + /* si */{ x, x, x, x, x }, }; // clang-format on for (std::size_t i = 0; i < n; ++i) { From cf67c0f26446ff2a46003ddfdd7cec54fb4da7e0 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 21:25:27 -0500 Subject: [PATCH 15/29] TODO: AMEND: trace_tags is a vector --- src/datadog/extracted_data.h | 5 +++-- src/datadog/tag_propagation.cpp | 11 +++++------ src/datadog/tag_propagation.h | 7 ++++--- src/datadog/trace_segment.cpp | 25 +++++++++++++++++++------ src/datadog/trace_segment.h | 6 +++--- src/datadog/tracer.cpp | 13 +++++++------ src/datadog/w3c_propagation.cpp | 8 ++++---- src/datadog/w3c_propagation.h | 2 +- test/matchers.h | 16 +++++++++++++++- test/test_span.cpp | 2 -- test/test_tracer.cpp | 2 +- test/test_tracer_config.cpp | 1 + 12 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index f6d90f82..0f182d7f 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -5,7 +5,8 @@ #include #include -#include +#include +#include #include "optional.h" @@ -16,7 +17,7 @@ struct ExtractedData { Optional trace_id; Optional parent_id; Optional origin; - std::unordered_map trace_tags; + std::vector> trace_tags; Optional sampling_priority; // If this `ExtractedData` was created on account of `PropagationStyle::W3C` // and trace context was successfully extracted, then `full_w3c_trace_id_hex` diff --git a/src/datadog/tag_propagation.cpp b/src/datadog/tag_propagation.cpp index 6c17be39..e3f024ec 100644 --- a/src/datadog/tag_propagation.cpp +++ b/src/datadog/tag_propagation.cpp @@ -32,7 +32,7 @@ namespace { // Insert into the specified `destination` a tag decoded from the specified // `entry`. Return an `Error` if an error occurs. Expected decode_tag( - std::unordered_map& destination, + std::vector>& destination, StringView entry) { const auto separator = std::find(entry.begin(), entry.end(), '='); if (separator == entry.end()) { @@ -44,8 +44,7 @@ Expected decode_tag( const StringView key = range(entry.begin(), separator); const StringView value = range(separator + 1, entry.end()); - // Among duplicate keys, most recent value wins. - destination.insert_or_assign(std::string(key), std::string(value)); + destination.emplace_back(std::string(key), std::string(value)); return nullopt; } @@ -59,9 +58,9 @@ void append_tag(std::string& serialized_tags, StringView tag_key, } // namespace -Expected> decode_tags( +Expected>> decode_tags( StringView header_value) { - std::unordered_map tags; + std::vector> tags; auto iter = header_value.begin(); const auto end = header_value.end(); @@ -88,7 +87,7 @@ Expected> decode_tags( } std::string encode_tags( - const std::unordered_map& trace_tags) { + const std::vector>& trace_tags) { std::string result; auto iter = trace_tags.begin(); if (iter == trace_tags.end()) { diff --git a/src/datadog/tag_propagation.h b/src/datadog/tag_propagation.h index b518716d..99666856 100644 --- a/src/datadog/tag_propagation.h +++ b/src/datadog/tag_propagation.h @@ -12,7 +12,8 @@ // "x-datadog-tags" header format. #include -#include +#include +#include #include "expected.h" #include "string_view.h" @@ -22,13 +23,13 @@ namespace tracing { // Return a name->value mapping of tags parsed from the specified // `header_value`, or return an `Error` if an error occurs. -Expected> decode_tags( +Expected>> decode_tags( StringView header_value); // Serialize the specified `trace_tags` into the propagation format and return // the resulting string. std::string encode_tags( - const std::unordered_map& trace_tags); + const std::vector>& trace_tags); } // namespace tracing } // namespace datadog diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 8b2ec50e..567259da 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "collector.h" #include "collector_response.h" @@ -30,7 +31,7 @@ namespace { // on the specified `local_root_tags`. void inject_trace_tags( DictWriter& writer, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, std::size_t tags_header_max_size, std::unordered_map& local_root_tags, Logger& logger) { @@ -63,7 +64,7 @@ TraceSegment::TraceSegment( const std::vector& injection_styles, const Optional& hostname, Optional origin, std::size_t tags_header_max_size, - std::unordered_map trace_tags, + std::vector> trace_tags, Optional sampling_decision, Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, @@ -224,11 +225,23 @@ void TraceSegment::update_decision_maker_trace_tag() { assert(sampling_decision_); + const auto found = std::find_if( + trace_tags_.begin(), trace_tags_.end(), [](const auto& entry) { + return entry.first == tags::internal::decision_maker; + }); + if (sampling_decision_->priority <= 0) { - trace_tags_.erase(tags::internal::decision_maker); + if (found != trace_tags_.end()) { + trace_tags_.erase(found); + } + return; + } + + auto value = "-" + std::to_string(*sampling_decision_->mechanism); + if (found == trace_tags_.end()) { + trace_tags_.emplace_back(tags::internal::decision_maker, std::move(value)); } else { - trace_tags_[tags::internal::decision_maker] = - "-" + std::to_string(*sampling_decision_->mechanism); + found->second == std::move(value); } } @@ -244,7 +257,7 @@ void TraceSegment::inject(DictWriter& writer, const SpanData& span) { // So, we lock here, make a sampling decision if necessary, and then copy the // decision and trace tags before unlocking. int sampling_priority; - std::unordered_map trace_tags; + std::vector> trace_tags; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); diff --git a/src/datadog/trace_segment.h b/src/datadog/trace_segment.h index 772e9c36..0a8ccc42 100644 --- a/src/datadog/trace_segment.h +++ b/src/datadog/trace_segment.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "expected.h" @@ -62,7 +62,7 @@ class TraceSegment { const Optional hostname_; const Optional origin_; const std::size_t tags_header_max_size_; - std::unordered_map trace_tags_; + std::vector> trace_tags_; std::vector> spans_; std::size_t num_finished_spans_; @@ -81,7 +81,7 @@ class TraceSegment { const std::vector& injection_styles, const Optional& hostname, Optional origin, std::size_t tags_header_max_size, - std::unordered_map trace_tags, + std::vector> trace_tags, Optional sampling_decision, Optional full_w3c_trace_id_hex, Optional additional_w3c_tracestate, diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 79d27dd3..dcc5c142 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -37,11 +37,12 @@ void handle_trace_tags(StringView trace_tags, ExtractedData& result, if (auto* error = maybe_trace_tags.if_error()) { logger.log_error(*error); span_tags[tags::internal::propagation_error] = "decoding_error"; - } else { - for (const auto& [key, value] : *maybe_trace_tags) { - if (starts_with(key, "_dd.p.")) { - result.trace_tags.insert_or_assign(key, value); - } + return; + } + + for (auto& [key, value] : *maybe_trace_tags) { + if (starts_with(key, "_dd.p.")) { + result.trace_tags.emplace_back(std::move(key), std::move(value)); } } } @@ -248,7 +249,7 @@ Span Tracer::create_span(const SpanConfig& config) { const auto segment = std::make_shared( logger_, collector_, trace_sampler_, span_sampler_, defaults_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, - std::unordered_map{} /* trace_tags */, + std::vector>{} /* trace_tags */, nullopt /* sampling_decision */, nullopt /* full_w3c_trace_id_hex */, nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data)); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 87935829..347c2140 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -221,8 +221,8 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { // transformation. std::string decoded_value{value}; std::replace(decoded_value.begin(), decoded_value.end(), '~', '='); - result.trace_tags.insert_or_assign(std::move(tag_name), - std::move(decoded_value)); + result.trace_tags.emplace_back(std::move(tag_name), + std::move(decoded_value)); } else { // Unrecognized key: append the whole pair to // `additional_datadog_w3c_tracestate`, which will be used if/when we @@ -325,7 +325,7 @@ std::string encode_traceparent( std::string encode_datadog_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate) { std::string result = "dd=s:"; result += std::to_string(sampling_priority); @@ -387,7 +387,7 @@ std::string encode_datadog_tracestate( std::string encode_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, const Optional& additional_w3c_tracestate) { std::string result = encode_datadog_tracestate( diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index fe11724e..57946ca9 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -38,7 +38,7 @@ std::string encode_traceparent( // Return a value for the "tracestate" header containing the specified fields. std::string encode_tracestate( int sampling_priority, const Optional& origin, - const std::unordered_map& trace_tags, + const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, const Optional& additional_w3c_tracestate); diff --git a/test/matchers.h b/test/matchers.h index 88102906..83b14d88 100644 --- a/test/matchers.h +++ b/test/matchers.h @@ -2,6 +2,7 @@ #include #include +#include #include "test.h" @@ -9,13 +10,26 @@ template class ContainsSubset : public Catch::MatcherBase { const Map* subset_; + // `find` for when we're comparing with a `vector`. + template + static auto find(const std::vector& other, const LookupKey& key) { + return std::find_if(other.begin(), other.end(), + [&](const auto& entry) { return entry.first == key; }); + } + + // `find` for when we're comparing with an associative container. + template + static auto find(const Other& other, const LookupKey& key) { + return other.find(key); + } + public: ContainsSubset(const Map& subset) : subset_(&subset) {} bool match(const Map& other) const override { return std::all_of(subset_->begin(), subset_->end(), [&](const auto& item) { const auto& [key, value] = item; - auto found = other.find(key); + auto found = find(other, key); return found != other.end() && found->second == value; }); } diff --git a/test/test_span.cpp b/test/test_span.cpp index be331201..0582bf20 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -434,8 +434,6 @@ TEST_CASE("injection") { span->inject(writer); REQUIRE(writer.items.at("x-datadog-origin") == "Egypt"); - // Trace tags could get reordered (because we parse them into a hash - // table). So, compare the parsed versions. REQUIRE(writer.items.count("x-datadog-tags") == 1); const auto output = decode_tags(writer.items.at("x-datadog-tags")); const auto input = decode_tags(trace_tags); diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 7142c17d..0ad69d04 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -667,7 +667,7 @@ TEST_CASE("span extraction") { Optional tracestate; Optional expected_sampling_priority = {}; Optional expected_origin = {}; - std::unordered_map expected_trace_tags = {}; + std::vector> expected_trace_tags = {}; Optional expected_additional_w3c_tracestate = {}; Optional expected_additional_datadog_w3c_tracestate = {}; }; diff --git a/test/test_tracer_config.cpp b/test/test_tracer_config.cpp index a106875f..ab583228 100644 --- a/test/test_tracer_config.cpp +++ b/test/test_tracer_config.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "mocks/collectors.h" #include "mocks/event_schedulers.h" From 0cff419166539811684cc492da186c4ade982eb8 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:00:08 -0500 Subject: [PATCH 16/29] TODO: AMEND: tracestate injection tested --- src/datadog/w3c_propagation.cpp | 25 ++++-------- test/test_span.cpp | 67 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index 347c2140..a376dd26 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -330,9 +330,6 @@ std::string encode_datadog_tracestate( std::string result = "dd=s:"; result += std::to_string(sampling_priority); - const std::size_t max_size = 256; - std::size_t last_good_size = result.size(); - if (origin) { result += ";o:"; result += *origin; @@ -340,12 +337,6 @@ std::string encode_datadog_tracestate( verboten(0x20, 0x7e, ",;="), '_'); } - if (result.size() > max_size) { - result.resize(last_good_size); - return result; - } - last_good_size = result.size(); - for (const auto& [key, value] : trace_tags) { const StringView prefix = "_dd.p."; if (!starts_with(key, prefix)) { @@ -365,12 +356,6 @@ std::string encode_datadog_tracestate( // `value` might contain equal signs ("="), which is reserved in tracestate. // Replace them with tildes ("~"). std::replace(result.end() - value.size(), result.end(), '=', '~'); - - if (result.size() > max_size) { - result.resize(last_good_size); - return result; - } - last_good_size = result.size(); } if (additional_datadog_w3c_tracestate) { @@ -378,8 +363,14 @@ std::string encode_datadog_tracestate( result += *additional_datadog_w3c_tracestate; } - if (result.size() > max_size) { - result.resize(last_good_size); + const std::size_t max_size = 256; + while (result.size() > max_size) { + const auto last_semicolon_index = result.rfind(';'); + // This assumption is safe, because `result` always begins with + // "dd=s:", and that's fewer than `max_size` characters for any + // ``. + assert(last_semicolon_index != std::string::npos); + result.resize(last_semicolon_index); } return result; diff --git a/test/test_span.cpp b/test/test_span.cpp index 0582bf20..ada7df6f 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -558,7 +558,7 @@ TEST_CASE("injecting W3C tracestate header") { // - special tilde ("~") behavior // - length limit: // - at origin - // - at a trace tag (extract extra fields from W3C) + // - at a trace tag // - at the extra fields (extracted from W3C) TracerConfig config; @@ -585,6 +585,9 @@ TEST_CASE("injecting W3C tracestate header") { std::string expected_tracestate; }; + static const auto traceparent_drop = + "00-00000000000000000000000000000001-0000000000000001-00"; + // clang-format off auto test_case = GENERATE(values({ {__LINE__, "sampling priority", @@ -600,9 +603,69 @@ TEST_CASE("injecting W3C tracestate header") { {__LINE__, "trace tags", {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, - {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y"}}, + {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y,ignored=wrong_prefix"}}, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;t.foo:x;t.bar:y"}, + + {__LINE__, "extra fields", + {{"traceparent", traceparent_drop}, {"tracestate", "dd=foo:bar;boing:boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;foo:bar;boing:boing"}, + + {__LINE__, "all of the above", + {{"traceparent", traceparent_drop}, + {"tracestate", "dd=o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"}, + + {__LINE__, "replace invalid characters in origin", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "France, is a country=nation; so is 台北."}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;o:France_ is a country_nation_ so is ______."}, + + {__LINE__, "replace invalid characters in trace tag key", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.a;d台北x =foo,_dd.p.ok=bar"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.a_d______x_:foo;t.ok:bar"}, + + {__LINE__, "replace invalid characters in trace tag value", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.wacky=hello fr~d; how are คุณ?"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.wacky:hello fr_d_ how are _________?"}, + + {__LINE__, "replace equal signs with tildes in trace tag value", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.base64_thingy=d2Fra2EhIHdhaw=="}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.base64_thingy:d2Fra2EhIHdhaw~~"}, + + {__LINE__, "oversized origin truncates it and subsequent fields", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-origin", "long cat is looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"}, + {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.honk=honk"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1"}, + + {__LINE__, "oversized trace tag truncates it and subsequent fields", + {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"}, + {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.long_cat_is=looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong,_dd.p.lost=forever"}}, + // The "s:-1" comes from the 0% sample rate. + "dd=s:-1;t.foo:bar"}, + + {__LINE__, "oversized extra field truncates itself and subsequent fields", + {{"traceparent", traceparent_drop}, + {"tracestate", "dd=foo:bar;long_cat_is:looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong;lost:forever"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0;foo:bar"}, + + {__LINE__, "non-Datadog tracestate", + {{"traceparent", traceparent_drop}, + {"tracestate", "foo=bar,boing=boing"}}, + // The "s:0" comes from the sampling decision in `traceparent_drop`. + "dd=s:0,foo=bar,boing=boing"}, })); // clang-format on From 4e8719dc59bb07a358789ebd3f498208b9bd29bc Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:04:11 -0500 Subject: [PATCH 17/29] TODO: AMEND: fix scary typo --- src/datadog/trace_segment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 567259da..11040a66 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -241,7 +241,7 @@ void TraceSegment::update_decision_maker_trace_tag() { if (found == trace_tags_.end()) { trace_tags_.emplace_back(tags::internal::decision_maker, std::move(value)); } else { - found->second == std::move(value); + found->second = std::move(value); } } From a72b2977d443ce507e398c1a8e006b23c822776d Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Wed, 21 Dec 2022 23:33:07 -0500 Subject: [PATCH 18/29] TODO: AMEND: update includes graph in doc/ --- doc/includes.dot | 673 +++++----- doc/includes.svg | 3362 ++++++++++++++++++++++++---------------------- 2 files changed, 2091 insertions(+), 1944 deletions(-) diff --git a/doc/includes.dot b/doc/includes.dot index 1725ee7f..a507fbdd 100644 --- a/doc/includes.dot +++ b/doc/includes.dot @@ -1,7 +1,7 @@ digraph G { - node0 [label="rate.cpp"]; - node1 [label="span_data.cpp"]; - node2 [label="threaded_event_scheduler.cpp"]; + node0 [label="threaded_event_scheduler.cpp"]; + node1 [label="rate.cpp"]; + node2 [label="span_data.cpp"]; node3 [label="default_http_client_null.cpp"]; node4 [label="trace_segment.h"]; node5 [label="tag_propagation.cpp"]; @@ -10,8 +10,8 @@ digraph G { node8 [label="string_view.h"]; node9 [label="span_sampler.h"]; node10 [label="dict_reader.cpp"]; - node11 [label="tags.h"]; - node12 [label="tracer_config.h"]; + node11 [label="tracer_config.h"]; + node12 [label="tags.h"]; node13 [label="curl/curl.h"]; node14 [label="expected.h"]; node15 [label="limiter.h"]; @@ -20,348 +20,369 @@ digraph G { node18 [label="sampling_mechanism.cpp"]; node19 [label="logger.cpp"]; node20 [label="rate.h"]; - node21 [label="sampling_util.h"]; - node22 [label="string_view.cpp"]; - node23 [label="sampling_decision.h"]; - node24 [label="span_data.h"]; - node25 [label="datadog_agent_config.h"]; - node26 [label="sampling_mechanism.h"]; - node27 [label="glob.h"]; - node28 [label="msgpack.cpp"]; - node29 [label="absl/types/optional.h"]; - node30 [label="default_http_client.h"]; - node31 [label="sampling_util.cpp"]; - node32 [label="datadog_agent_config.cpp"]; - node33 [label="span.cpp"]; - node34 [label="event_scheduler.cpp"]; - node35 [label="json_fwd.hpp"]; - node36 [label="datadog_agent.h"]; - node37 [label="propagation_style.cpp"]; - node38 [label="error.h"]; - node39 [label="event_scheduler.h"]; - node40 [label="tag_propagation.h"]; - node41 [label="clock.h"]; - node42 [label="logger.h"]; - node43 [label="span_defaults.h"]; - node44 [label="curl.h"]; - node45 [label="span_defaults.cpp"]; - node46 [label="threaded_event_scheduler.h"]; - node47 [label="tracer.h"]; - node48 [label="parse_util.h"]; - node49 [label="expected.cpp"]; - node50 [label="collector.cpp"]; - node51 [label="id_generator.cpp"]; - node52 [label="trace_sampler.cpp"]; - node53 [label="sampling_decision.cpp"]; - node54 [label="sampling_priority.cpp"]; - node55 [label="span_sampler.cpp"]; - node56 [label="collector.h"]; - node57 [label="curl.cpp"]; - node58 [label="null_collector.cpp"]; - node59 [label="cerr_logger.cpp"]; - node60 [label="collector_response.h"]; - node61 [label="msgpack.h"]; - node62 [label="trace_sampler.h"]; - node63 [label="span_config.cpp"]; - node64 [label="propagation_style.h"]; - node65 [label="dict_writer.cpp"]; - node66 [label="trace_sampler_config.cpp"]; - node67 [label="environment.h"]; - node68 [label="span_config.h"]; - node69 [label="datadog_agent.cpp"]; - node70 [label="winsock.h"]; - node71 [label="environment.cpp"]; - node72 [label="id_generator.h"]; - node73 [label="tracer_config.cpp"]; - node74 [label="net_util.cpp"]; - node75 [label="parse_util.cpp"]; - node76 [label="http_client.cpp"]; - node77 [label="optional.h"]; - node78 [label="null_collector.h"]; - node79 [label="charconv"]; - node80 [label="version.h"]; - node81 [label="span_sampler_config.h"]; - node82 [label="span_matcher.h"]; - node83 [label="version.cpp"]; - node84 [label="dict_reader.h"]; - node85 [label="span_matcher.cpp"]; - node86 [label="tracer.cpp"]; - node87 [label="default_http_client_curl.cpp"]; - node88 [label="http_client.h"]; - node89 [label="clock.cpp"]; - node90 [label="collector_response.cpp"]; - node91 [label="sampling_priority.h"]; - node92 [label="net_util.h"]; - node93 [label="limiter.cpp"]; - node94 [label="error.cpp"]; - node95 [label="absl/strings/string_view.h"]; - node96 [label="trace_sampler_config.h"]; - node97 [label="tags.cpp"]; - node98 [label="cerr_logger.h"]; - node99 [label="trace_segment.cpp"]; - node100 [label="span_sampler_config.cpp"]; + node21 [label="hex.cpp"]; + node22 [label="sampling_util.h"]; + node23 [label="string_view.cpp"]; + node24 [label="datadog_agent_config.h"]; + node25 [label="span_data.h"]; + node26 [label="sampling_decision.h"]; + node27 [label="sampling_mechanism.h"]; + node28 [label="glob.h"]; + node29 [label="msgpack.cpp"]; + node30 [label="absl/types/optional.h"]; + node31 [label="default_http_client.h"]; + node32 [label="sampling_util.cpp"]; + node33 [label="datadog_agent_config.cpp"]; + node34 [label="span.cpp"]; + node35 [label="event_scheduler.cpp"]; + node36 [label="json_fwd.hpp"]; + node37 [label="datadog_agent.h"]; + node38 [label="propagation_style.cpp"]; + node39 [label="error.h"]; + node40 [label="event_scheduler.h"]; + node41 [label="tag_propagation.h"]; + node42 [label="clock.h"]; + node43 [label="logger.h"]; + node44 [label="span_defaults.h"]; + node45 [label="curl.h"]; + node46 [label="span_defaults.cpp"]; + node47 [label="threaded_event_scheduler.h"]; + node48 [label="tracer.h"]; + node49 [label="parse_util.h"]; + node50 [label="expected.cpp"]; + node51 [label="collector.cpp"]; + node52 [label="extracted_data.cpp"]; + node53 [label="id_generator.cpp"]; + node54 [label="trace_sampler.cpp"]; + node55 [label="sampling_decision.cpp"]; + node56 [label="extracted_data.h"]; + node57 [label="sampling_priority.cpp"]; + node58 [label="collector.h"]; + node59 [label="span_sampler.cpp"]; + node60 [label="curl.cpp"]; + node61 [label="null_collector.cpp"]; + node62 [label="cerr_logger.cpp"]; + node63 [label="collector_response.h"]; + node64 [label="w3c_propagation.h"]; + node65 [label="msgpack.h"]; + node66 [label="trace_sampler.h"]; + node67 [label="span_config.cpp"]; + node68 [label="propagation_style.h"]; + node69 [label="dict_writer.cpp"]; + node70 [label="trace_sampler_config.cpp"]; + node71 [label="environment.h"]; + node72 [label="span_config.h"]; + node73 [label="datadog_agent.cpp"]; + node74 [label="w3c_propagation.cpp"]; + node75 [label="winsock.h"]; + node76 [label="environment.cpp"]; + node77 [label="id_generator.h"]; + node78 [label="tracer_config.cpp"]; + node79 [label="net_util.cpp"]; + node80 [label="parse_util.cpp"]; + node81 [label="http_client.cpp"]; + node82 [label="optional.h"]; + node83 [label="null_collector.h"]; + node84 [label="charconv"]; + node85 [label="version.h"]; + node86 [label="version.cpp"]; + node87 [label="span_matcher.h"]; + node88 [label="span_sampler_config.h"]; + node89 [label="dict_reader.h"]; + node90 [label="span_matcher.cpp"]; + node91 [label="tracer.cpp"]; + node92 [label="http_client.h"]; + node93 [label="default_http_client_curl.cpp"]; + node94 [label="clock.cpp"]; + node95 [label="sampling_priority.h"]; + node96 [label="collector_response.cpp"]; + node97 [label="hex.h"]; + node98 [label="limiter.cpp"]; + node99 [label="net_util.h"]; + node100 [label="error.cpp"]; + node101 [label="absl/strings/string_view.h"]; + node102 [label="trace_sampler_config.h"]; + node103 [label="tags.cpp"]; + node104 [label="cerr_logger.h"]; + node105 [label="trace_segment.cpp"]; + node106 [label="span_sampler_config.cpp"]; subgraph U { edge [dir=none]; } subgraph D { - node0 -> node20 []; - node1 -> node43 []; - node1 -> node68 []; - node1 -> node8 []; - node1 -> node24 []; - node1 -> node61 []; - node1 -> node11 []; - node1 -> node38 []; - node2 -> node7 []; - node2 -> node46 []; - node3 -> node30 []; - node4 -> node64 []; - node4 -> node23 []; + node0 -> node7 []; + node0 -> node47 []; + node1 -> node20 []; + node2 -> node44 []; + node2 -> node72 []; + node2 -> node8 []; + node2 -> node25 []; + node2 -> node65 []; + node2 -> node12 []; + node2 -> node39 []; + node3 -> node31 []; + node4 -> node68 []; + node4 -> node26 []; node4 -> node14 []; - node4 -> node77 []; - node5 -> node40 []; - node5 -> node48 []; - node5 -> node38 []; + node4 -> node82 []; + node5 -> node41 []; + node5 -> node49 []; + node5 -> node39 []; node6 -> node8 []; - node8 -> node95 []; - node9 -> node41 []; - node9 -> node81 []; - node9 -> node23 []; + node8 -> node101 []; + node9 -> node42 []; + node9 -> node88 []; + node9 -> node26 []; node9 -> node15 []; - node9 -> node35 []; - node10 -> node84 []; - node11 -> node8 []; - node12 -> node43 []; - node12 -> node64 []; - node12 -> node81 []; - node12 -> node25 []; - node12 -> node96 []; - node12 -> node14 []; - node12 -> node38 []; - node14 -> node77 []; - node14 -> node38 []; - node15 -> node41 []; + node9 -> node36 []; + node10 -> node89 []; + node11 -> node44 []; + node11 -> node68 []; + node11 -> node88 []; + node11 -> node24 []; + node11 -> node102 []; + node11 -> node14 []; + node11 -> node39 []; + node12 -> node8 []; + node14 -> node82 []; + node14 -> node39 []; + node15 -> node42 []; node15 -> node20 []; - node16 -> node77 []; - node17 -> node41 []; - node17 -> node72 []; - node17 -> node8 []; + node16 -> node82 []; + node17 -> node42 []; node17 -> node77 []; - node17 -> node38 []; - node18 -> node26 []; - node19 -> node42 []; - node19 -> node38 []; + node17 -> node8 []; + node17 -> node82 []; + node17 -> node39 []; + node18 -> node27 []; + node19 -> node43 []; + node19 -> node39 []; node20 -> node14 []; - node21 -> node20 []; - node22 -> node8 []; - node23 -> node26 []; - node23 -> node20 []; - node23 -> node77 []; - node24 -> node41 []; + node21 -> node97 []; + node22 -> node20 []; + node23 -> node8 []; node24 -> node14 []; + node24 -> node92 []; node24 -> node8 []; - node24 -> node77 []; + node25 -> node42 []; node25 -> node14 []; - node25 -> node88 []; node25 -> node8 []; - node27 -> node8 []; - node28 -> node61 []; - node28 -> node38 []; - node31 -> node21 []; - node32 -> node48 []; - node32 -> node25 []; - node32 -> node30 []; - node32 -> node67 []; - node32 -> node46 []; - node33 -> node4 []; - node33 -> node68 []; - node33 -> node6 []; + node25 -> node82 []; + node26 -> node27 []; + node26 -> node20 []; + node26 -> node82 []; + node28 -> node8 []; + node29 -> node65 []; + node29 -> node39 []; + node32 -> node22 []; + node33 -> node49 []; node33 -> node24 []; - node33 -> node77 []; - node33 -> node11 []; - node33 -> node8 []; - node33 -> node17 []; - node34 -> node39 []; - node36 -> node41 []; - node36 -> node56 []; - node36 -> node88 []; - node36 -> node39 []; - node37 -> node64 []; - node37 -> node7 []; - node38 -> node8 []; - node39 -> node35 []; - node39 -> node38 []; - node40 -> node14 []; - node40 -> node8 []; - node42 -> node8 []; - node43 -> node35 []; - node44 -> node88 []; - node44 -> node35 []; - node45 -> node43 []; - node45 -> node7 []; - node46 -> node39 []; - node47 -> node41 []; - node47 -> node72 []; - node47 -> node17 []; - node47 -> node77 []; - node47 -> node38 []; - node47 -> node14 []; - node47 -> node12 []; + node33 -> node31 []; + node33 -> node71 []; + node33 -> node47 []; + node34 -> node4 []; + node34 -> node72 []; + node34 -> node6 []; + node34 -> node25 []; + node34 -> node82 []; + node34 -> node12 []; + node34 -> node8 []; + node34 -> node17 []; + node35 -> node40 []; + node37 -> node42 []; + node37 -> node58 []; + node37 -> node92 []; + node37 -> node40 []; + node38 -> node68 []; + node38 -> node7 []; + node39 -> node8 []; + node40 -> node36 []; + node40 -> node39 []; + node41 -> node14 []; + node41 -> node8 []; + node43 -> node8 []; + node44 -> node36 []; + node45 -> node92 []; + node45 -> node36 []; + node46 -> node44 []; + node46 -> node7 []; + node47 -> node40 []; + node48 -> node42 []; + node48 -> node77 []; + node48 -> node17 []; + node48 -> node82 []; + node48 -> node39 []; node48 -> node14 []; - node48 -> node8 []; + node48 -> node11 []; node49 -> node14 []; - node50 -> node56 []; - node51 -> node72 []; - node52 -> node21 []; - node52 -> node62 []; - node52 -> node91 []; - node52 -> node60 []; - node52 -> node24 []; - node52 -> node23 []; - node52 -> node7 []; - node53 -> node23 []; - node54 -> node91 []; + node49 -> node8 []; + node50 -> node14 []; + node51 -> node58 []; + node52 -> node56 []; + node53 -> node77 []; + node54 -> node22 []; + node54 -> node66 []; + node54 -> node95 []; + node54 -> node63 []; + node54 -> node25 []; + node54 -> node26 []; + node54 -> node7 []; node55 -> node26 []; - node55 -> node21 []; - node55 -> node91 []; - node55 -> node7 []; - node55 -> node9 []; - node55 -> node24 []; - node56 -> node14 []; - node56 -> node77 []; - node56 -> node35 []; - node57 -> node42 []; - node57 -> node7 []; - node57 -> node8 []; - node57 -> node44 []; - node57 -> node6 []; - node57 -> node48 []; - node57 -> node13 []; - node57 -> node88 []; - node57 -> node84 []; - node58 -> node7 []; - node58 -> node78 []; - node59 -> node98 []; - node60 -> node20 []; + node56 -> node82 []; + node57 -> node95 []; + node58 -> node14 []; + node58 -> node82 []; + node58 -> node36 []; + node59 -> node27 []; + node59 -> node22 []; + node59 -> node95 []; + node59 -> node7 []; + node59 -> node9 []; + node59 -> node25 []; + node60 -> node43 []; + node60 -> node7 []; node60 -> node8 []; - node61 -> node14 []; - node61 -> node8 []; - node62 -> node41 []; - node62 -> node96 []; - node62 -> node20 []; - node62 -> node15 []; - node62 -> node77 []; - node62 -> node35 []; - node63 -> node68 []; - node64 -> node35 []; - node65 -> node6 []; - node66 -> node48 []; - node66 -> node96 []; - node66 -> node7 []; - node66 -> node67 []; - node67 -> node8 []; - node67 -> node77 []; - node67 -> node35 []; - node68 -> node41 []; - node68 -> node77 []; - node69 -> node42 []; - node69 -> node25 []; - node69 -> node60 []; - node69 -> node7 []; - node69 -> node80 []; - node69 -> node36 []; - node69 -> node62 []; + node60 -> node45 []; + node60 -> node6 []; + node60 -> node49 []; + node60 -> node13 []; + node60 -> node92 []; + node60 -> node89 []; + node61 -> node7 []; + node61 -> node83 []; + node62 -> node104 []; + node63 -> node20 []; + node63 -> node8 []; + node64 -> node56 []; + node64 -> node14 []; + node64 -> node82 []; + node65 -> node14 []; + node65 -> node8 []; + node66 -> node42 []; + node66 -> node102 []; + node66 -> node20 []; + node66 -> node15 []; + node66 -> node82 []; + node66 -> node36 []; + node67 -> node72 []; + node68 -> node36 []; node69 -> node6 []; - node69 -> node24 []; - node69 -> node61 []; - node71 -> node7 []; - node71 -> node67 []; - node73 -> node48 []; - node73 -> node98 []; + node70 -> node49 []; + node70 -> node102 []; + node70 -> node7 []; + node70 -> node71 []; + node71 -> node8 []; + node71 -> node82 []; + node71 -> node36 []; + node72 -> node42 []; + node72 -> node82 []; + node73 -> node43 []; + node73 -> node24 []; + node73 -> node63 []; node73 -> node7 []; - node73 -> node12 []; - node73 -> node67 []; - node73 -> node8 []; - node73 -> node78 []; - node73 -> node36 []; - node74 -> node92 []; - node74 -> node70 []; - node75 -> node79 []; - node75 -> node48 []; - node75 -> node38 []; - node76 -> node88 []; - node77 -> node29 []; - node78 -> node56 []; - node80 -> node8 []; - node81 -> node14 []; - node81 -> node20 []; - node81 -> node82 []; - node81 -> node77 []; - node81 -> node35 []; - node82 -> node14 []; - node82 -> node35 []; - node83 -> node80 []; - node84 -> node8 []; - node84 -> node77 []; - node85 -> node7 []; - node85 -> node24 []; - node85 -> node38 []; - node85 -> node82 []; - node85 -> node77 []; - node86 -> node4 []; - node86 -> node68 []; - node86 -> node47 []; - node86 -> node9 []; - node86 -> node36 []; - node86 -> node11 []; - node86 -> node40 []; - node86 -> node42 []; - node86 -> node62 []; - node86 -> node48 []; - node86 -> node7 []; - node86 -> node92 []; - node86 -> node24 []; - node86 -> node84 []; - node86 -> node67 []; - node86 -> node17 []; - node86 -> node80 []; - node87 -> node30 []; - node87 -> node44 []; + node73 -> node85 []; + node73 -> node37 []; + node73 -> node66 []; + node73 -> node6 []; + node73 -> node25 []; + node73 -> node65 []; + node74 -> node97 []; + node74 -> node89 []; + node74 -> node12 []; + node74 -> node64 []; + node74 -> node49 []; + node76 -> node7 []; + node76 -> node71 []; + node78 -> node49 []; + node78 -> node104 []; + node78 -> node7 []; + node78 -> node11 []; + node78 -> node71 []; + node78 -> node8 []; + node78 -> node83 []; + node78 -> node37 []; + node79 -> node99 []; + node79 -> node75 []; + node80 -> node84 []; + node80 -> node49 []; + node80 -> node39 []; + node81 -> node92 []; + node82 -> node30 []; + node83 -> node58 []; + node85 -> node8 []; + node86 -> node85 []; + node87 -> node14 []; + node87 -> node36 []; node88 -> node14 []; - node88 -> node38 []; - node88 -> node77 []; - node88 -> node35 []; - node89 -> node41 []; - node90 -> node60 []; - node91 -> node26 []; - node92 -> node77 []; - node93 -> node15 []; - node94 -> node38 []; - node96 -> node14 []; - node96 -> node20 []; - node96 -> node82 []; - node96 -> node77 []; - node96 -> node35 []; - node97 -> node48 []; - node97 -> node11 []; - node98 -> node42 []; - node99 -> node56 []; - node99 -> node60 []; - node99 -> node9 []; - node99 -> node24 []; - node99 -> node77 []; - node99 -> node40 []; - node99 -> node42 []; - node99 -> node4 []; - node99 -> node79 []; - node99 -> node62 []; - node99 -> node6 []; - node99 -> node11 []; - node99 -> node38 []; - node100 -> node42 []; - node100 -> node81 []; - node100 -> node7 []; - node100 -> node14 []; - node100 -> node67 []; + node88 -> node20 []; + node88 -> node87 []; + node88 -> node82 []; + node88 -> node36 []; + node89 -> node8 []; + node89 -> node82 []; + node90 -> node7 []; + node90 -> node25 []; + node90 -> node39 []; + node90 -> node87 []; + node90 -> node82 []; + node91 -> node4 []; + node91 -> node48 []; + node91 -> node9 []; + node91 -> node37 []; + node91 -> node12 []; + node91 -> node41 []; + node91 -> node43 []; + node91 -> node72 []; + node91 -> node56 []; + node91 -> node64 []; + node91 -> node66 []; + node91 -> node49 []; + node91 -> node7 []; + node91 -> node99 []; + node91 -> node25 []; + node91 -> node89 []; + node91 -> node71 []; + node91 -> node17 []; + node91 -> node85 []; + node92 -> node14 []; + node92 -> node39 []; + node92 -> node82 []; + node92 -> node36 []; + node93 -> node31 []; + node93 -> node45 []; + node94 -> node42 []; + node95 -> node27 []; + node96 -> node63 []; + node97 -> node84 []; + node98 -> node15 []; + node99 -> node82 []; + node100 -> node39 []; + node102 -> node14 []; + node102 -> node20 []; + node102 -> node87 []; + node102 -> node82 []; + node102 -> node36 []; + node103 -> node49 []; + node103 -> node12 []; + node104 -> node43 []; + node105 -> node4 []; + node105 -> node97 []; + node105 -> node58 []; + node105 -> node63 []; + node105 -> node9 []; + node105 -> node25 []; + node105 -> node82 []; + node105 -> node41 []; + node105 -> node43 []; + node105 -> node64 []; + node105 -> node66 []; + node105 -> node6 []; + node105 -> node12 []; + node105 -> node39 []; + node106 -> node43 []; + node106 -> node88 []; + node106 -> node7 []; + node106 -> node14 []; + node106 -> node71 []; } } diff --git a/doc/includes.svg b/doc/includes.svg index 5e55705b..11e0a44e 100644 --- a/doc/includes.svg +++ b/doc/includes.svg @@ -4,2170 +4,2296 @@ - - + + G - + node0 - -rate.cpp + +threaded_event_scheduler.cpp - - -node20 - -rate.h + + +node7 + +json.hpp - + -node0->node20 - - +node0->node7 + + + + + +node47 + +threaded_event_scheduler.h + + + +node0->node47 + + node1 - -span_data.cpp - - - -node8 - -string_view.h + +rate.cpp - - -node1->node8 - - - - - -node11 - -tags.h + + +node20 + +rate.h - - -node1->node11 - - + + +node1->node20 + + - - -node24 - -span_data.h + + +node2 + +span_data.cpp - - -node1->node24 - - + + +node8 + +string_view.h - - -node38 - -error.h + + +node2->node8 + + - - -node1->node38 - - + + +node12 + +tags.h - - -node43 - -span_defaults.h + + +node2->node12 + + - - -node1->node43 - - + + +node25 + +span_data.h - - -node61 - -msgpack.h + + +node2->node25 + + - - -node1->node61 - - + + +node39 + +error.h - - -node68 - -span_config.h + + +node2->node39 + + - - -node1->node68 - - + + +node44 + +span_defaults.h - - -node2 - -threaded_event_scheduler.cpp + + +node2->node44 + + - - -node7 - -json.hpp + + +node65 + +msgpack.h - - -node2->node7 - - + + +node2->node65 + + - - -node46 - -threaded_event_scheduler.h + + +node72 + +span_config.h - - -node2->node46 - - + + +node2->node72 + + node3 - -default_http_client_null.cpp + +default_http_client_null.cpp - - -node30 - -default_http_client.h + + +node31 + +default_http_client.h - + -node3->node30 - - +node3->node31 + + node4 - -trace_segment.h + +trace_segment.h node14 - -expected.h + +expected.h node4->node14 - - + + - - -node23 - -sampling_decision.h + + +node26 + +sampling_decision.h - + -node4->node23 - - +node4->node26 + + - - -node64 - -propagation_style.h + + +node68 + +propagation_style.h - + -node4->node64 - - +node4->node68 + + - - -node77 - -optional.h + + +node82 + +optional.h - + -node4->node77 - - +node4->node82 + + node5 - -tag_propagation.cpp + +tag_propagation.cpp - + -node5->node38 - - +node5->node39 + + - - -node40 - -tag_propagation.h + + +node41 + +tag_propagation.h - + -node5->node40 - - +node5->node41 + + - - -node48 - -parse_util.h + + +node49 + +parse_util.h - + -node5->node48 - - +node5->node49 + + node6 - -dict_writer.h + +dict_writer.h node6->node8 - - + + - - -node95 - -absl/strings/string_view.h + + +node101 + +absl/strings/string_view.h - + -node8->node95 - - +node8->node101 + + node9 - -span_sampler.h + +span_sampler.h node15 - -limiter.h + +limiter.h node9->node15 - - + + - + -node9->node23 - - +node9->node26 + + - - -node35 - -json_fwd.hpp + + +node36 + +json_fwd.hpp - + -node9->node35 - - +node9->node36 + + - - -node41 - -clock.h + + +node42 + +clock.h - + -node9->node41 - - +node9->node42 + + - - -node81 - -span_sampler_config.h + + +node88 + +span_sampler_config.h - + -node9->node81 - - +node9->node88 + + node10 - -dict_reader.cpp + +dict_reader.cpp - - -node84 - -dict_reader.h + + +node89 + +dict_reader.h - + -node10->node84 - - +node10->node89 + + - - -node11->node8 - - + + +node11 + +tracer_config.h - - -node12 - -tracer_config.h + + +node11->node14 + + - - -node12->node14 - - + + +node24 + +datadog_agent_config.h - - -node25 - -datadog_agent_config.h + + +node11->node24 + + - - -node12->node25 - - + + +node11->node39 + + - - -node12->node38 - - + + +node11->node44 + + - + -node12->node43 - - +node11->node68 + + - + -node12->node64 - - +node11->node88 + + - - -node12->node81 - - + + +node102 + +trace_sampler_config.h - - -node96 - -trace_sampler_config.h + + +node11->node102 + + - - -node12->node96 - - + + +node12->node8 + + node13 - -curl/curl.h + +curl/curl.h - + -node14->node38 - - +node14->node39 + + - + -node14->node77 - - +node14->node82 + + node15->node20 - - + + - + -node15->node41 - - +node15->node42 + + node16 - -optional.cpp + +optional.cpp - + -node16->node77 - - +node16->node82 + + node17 - -span.h + +span.h node17->node8 - - + + - + -node17->node38 - - +node17->node39 + + - + -node17->node41 - - +node17->node42 + + - - -node72 - -id_generator.h + + +node77 + +id_generator.h - + -node17->node72 - - +node17->node77 + + - + -node17->node77 - - +node17->node82 + + node18 - -sampling_mechanism.cpp + +sampling_mechanism.cpp - - -node26 - -sampling_mechanism.h + + +node27 + +sampling_mechanism.h - + -node18->node26 - - +node18->node27 + + node19 - -logger.cpp + +logger.cpp - + -node19->node38 - - +node19->node39 + + - - -node42 - -logger.h + + +node43 + +logger.h - + -node19->node42 - - +node19->node43 + + node20->node14 - - + + node21 - -sampling_util.h + +hex.cpp - + + +node97 + +hex.h + + -node21->node20 - - +node21->node97 + + node22 - -string_view.cpp + +sampling_util.h - + -node22->node8 - - +node22->node20 + + - - -node23->node20 - - + + +node23 + +string_view.cpp - + -node23->node26 - - - - - -node23->node77 - - +node23->node8 + + - + node24->node8 - - + + - + node24->node14 - - + + - - -node24->node41 - - + + +node92 + +http_client.h - - -node24->node77 - - + + +node24->node92 + + - + node25->node8 - - + + - + node25->node14 - - + + - - -node88 - -http_client.h + + +node25->node42 + + - - -node25->node88 - - + + +node25->node82 + + - - -node27 - -glob.h + + +node26->node20 + + + + + +node26->node27 + + - + -node27->node8 - - +node26->node82 + + node28 - -msgpack.cpp + +glob.h - - -node28->node38 - - - - + -node28->node61 - - +node28->node8 + + node29 - -absl/types/optional.h - - - -node31 - -sampling_util.cpp + +msgpack.cpp - + -node31->node21 - - +node29->node39 + + + + + +node29->node65 + + + + + +node30 + +absl/types/optional.h node32 - -datadog_agent_config.cpp + +sampling_util.cpp - - -node32->node25 - - + + +node32->node22 + + + + + +node33 + +datadog_agent_config.cpp - + -node32->node30 - - +node33->node24 + + - - -node32->node46 - - - - - -node32->node48 - - - - - -node67 - -environment.h - - + -node32->node67 - - - - - -node33 - -span.cpp +node33->node31 + + - + -node33->node4 - - +node33->node47 + + - - -node33->node6 - - - - - -node33->node8 - - + + +node33->node49 + + - - -node33->node11 - - + + +node71 + +environment.h - - -node33->node17 - - + + +node33->node71 + + - - -node33->node24 - - + + +node34 + +span.cpp - + -node33->node68 - - +node34->node4 + + - - -node33->node77 - - + + +node34->node6 + + - - -node34 - -event_scheduler.cpp + + +node34->node8 + + - - -node39 - -event_scheduler.h + + +node34->node12 + + - + -node34->node39 - - +node34->node17 + + - - -node36 - -datadog_agent.h + + +node34->node25 + + - - -node36->node39 - - + + +node34->node72 + + - - -node36->node41 - - + + +node34->node82 + + - - -node56 - -collector.h + + +node35 + +event_scheduler.cpp - - -node36->node56 - - + + +node40 + +event_scheduler.h - - -node36->node88 - - + + +node35->node40 + + node37 - -propagation_style.cpp - - - -node37->node7 - - + +datadog_agent.h - + -node37->node64 - - +node37->node40 + + - + + +node37->node42 + + + + + +node58 + +collector.h + + + +node37->node58 + + + + + +node37->node92 + + + + + +node38 + +propagation_style.cpp + + -node38->node8 - - +node38->node7 + + + + + +node38->node68 + + - + -node39->node35 - - +node39->node8 + + - + -node39->node38 - - - - - -node40->node8 - - +node40->node36 + + - + -node40->node14 - - +node40->node39 + + - + -node42->node8 - - - - - -node43->node35 - - +node41->node8 + + - - -node44 - -curl.h + + +node41->node14 + + - - -node44->node35 - - + + +node43->node8 + + - + -node44->node88 - - +node44->node36 + + node45 - -span_defaults.cpp - - - -node45->node7 - - + +curl.h - + -node45->node43 - - +node45->node36 + + - - -node46->node39 - - + + +node45->node92 + + - - -node47 - -tracer.h + + +node46 + +span_defaults.cpp - - -node47->node12 - - + + +node46->node7 + + - - -node47->node14 - - + + +node46->node44 + + - - -node47->node17 - - + + +node47->node40 + + - - -node47->node38 - - + + +node48 + +tracer.h - - -node47->node41 - - + + +node48->node11 + + - - -node47->node72 - - + + +node48->node14 + + - + -node47->node77 - - +node48->node17 + + - - -node48->node8 - - + + +node48->node39 + + - - -node48->node14 - - + + +node48->node42 + + - - -node49 - -expected.cpp + + +node48->node77 + + - + + +node48->node82 + + + + +node49->node8 + + + + + node49->node14 - - + + node50 - -collector.cpp + +expected.cpp - + -node50->node56 - - +node50->node14 + + node51 - -id_generator.cpp + +collector.cpp - + -node51->node72 - - +node51->node58 + + node52 - -trace_sampler.cpp + +extracted_data.cpp - - -node52->node7 - - + + +node56 + +extracted_data.h - + -node52->node21 - - - - - -node52->node23 - - - - - -node52->node24 - - - - - -node60 - -collector_response.h - - - -node52->node60 - - - - - -node62 - -trace_sampler.h - - - -node52->node62 - - - - - -node91 - -sampling_priority.h - - - -node52->node91 - - +node52->node56 + + node53 - -sampling_decision.cpp + +id_generator.cpp - - -node53->node23 - - + + +node53->node77 + + node54 - -sampling_priority.cpp + +trace_sampler.cpp - + -node54->node91 - - +node54->node7 + + - - -node55 - -span_sampler.cpp + + +node54->node22 + + - - -node55->node7 - - + + +node54->node25 + + - - -node55->node9 - - + + +node54->node26 + + - - -node55->node21 - - + + +node63 + +collector_response.h - - -node55->node24 - - + + +node54->node63 + + + + + +node66 + +trace_sampler.h + + + +node54->node66 + + + + + +node95 + +sampling_priority.h + + + +node54->node95 + + + + + +node55 + +sampling_decision.cpp node55->node26 - - + + + + + +node56->node82 + + + + + +node57 + +sampling_priority.cpp - + -node55->node91 - - +node57->node95 + + - - -node56->node14 - - + + +node58->node14 + + - - -node56->node35 - - + + +node58->node36 + + - - -node56->node77 - - + + +node58->node82 + + - - -node57 - -curl.cpp + + +node59 + +span_sampler.cpp - - -node57->node6 - - + + +node59->node7 + + - + -node57->node7 - - +node59->node9 + + + + + +node59->node22 + + - + -node57->node8 - - +node59->node25 + + - - -node57->node13 - - + + +node59->node27 + + - - -node57->node42 - - + + +node59->node95 + + - - -node57->node44 - - + + +node60 + +curl.cpp + + + +node60->node6 + + + + + +node60->node7 + + - + -node57->node48 - - +node60->node8 + + - - -node57->node84 - - + + +node60->node13 + + - - -node57->node88 - - + + +node60->node43 + + - - -node58 - -null_collector.cpp + + +node60->node45 + + - - -node58->node7 - - + + +node60->node49 + + - - -node78 - -null_collector.h + + +node60->node89 + + - + -node58->node78 - - +node60->node92 + + - - -node59 - -cerr_logger.cpp + + +node61 + +null_collector.cpp - - -node98 - -cerr_logger.h + + +node61->node7 + + - - -node59->node98 - - + + +node83 + +null_collector.h - + -node60->node8 - - +node61->node83 + + - - -node60->node20 - - + + +node62 + +cerr_logger.cpp - - -node61->node8 - - + + +node104 + +cerr_logger.h - + -node61->node14 - - +node62->node104 + + - - -node62->node15 - - - - - -node62->node20 - - + + +node63->node8 + + - - -node62->node35 - - + + +node63->node20 + + - - -node62->node41 - - + + +node64 + +w3c_propagation.h - - -node62->node77 - - + + +node64->node14 + + - + -node62->node96 - - +node64->node56 + + - - -node63 - -span_config.cpp + + +node64->node82 + + - - -node63->node68 - - + + +node65->node8 + + - - -node64->node35 - - + + +node65->node14 + + - - -node65 - -dict_writer.cpp + + +node66->node15 + + - + -node65->node6 - - - - - -node66 - -trace_sampler_config.cpp +node66->node20 + + - + -node66->node7 - - +node66->node36 + + - - -node66->node48 - - - - - -node66->node67 - - + + +node66->node42 + + - + -node66->node96 - - - - - -node67->node8 - - +node66->node82 + + - - -node67->node35 - - + + +node66->node102 + + - - -node67->node77 - - + + +node67 + +span_config.cpp - - -node68->node41 - - + + +node67->node72 + + - - -node68->node77 - - + + +node68->node36 + + node69 - -datadog_agent.cpp + +dict_writer.cpp - + node69->node6 - - - - - -node69->node7 - - + + - - -node69->node24 - - + + +node70 + +trace_sampler_config.cpp - - -node69->node25 - - + + +node70->node7 + + - - -node69->node36 - - + + +node70->node49 + + - + -node69->node42 - - - - - -node69->node60 - - - - - -node69->node61 - - - - - -node69->node62 - - +node70->node71 + + - - -node80 - -version.h + + +node70->node102 + + - - -node69->node80 - - + + +node71->node8 + + - - -node70 - -winsock.h + + +node71->node36 + + - - -node71 - -environment.cpp + + +node71->node82 + + - - -node71->node7 - - + + +node72->node42 + + - - -node71->node67 - - + + +node72->node82 + + node73 - -tracer_config.cpp + +datadog_agent.cpp + + + +node73->node6 + + - + node73->node7 - - + + - - -node73->node8 - - + + +node73->node24 + + - - -node73->node12 - - + + +node73->node25 + + - - -node73->node36 - - + + +node73->node37 + + - - -node73->node48 - - + + +node73->node43 + + - - -node73->node67 - - + + +node73->node63 + + - - -node73->node78 - - + + +node73->node65 + + - - -node73->node98 - - + + +node73->node66 + + + + + +node85 + +version.h + + + +node73->node85 + + node74 - -net_util.cpp - - - -node74->node70 - - + +w3c_propagation.cpp - - -node92 - -net_util.h + + +node74->node12 + + - + -node74->node92 - - +node74->node49 + + + + + +node74->node64 + + + + + +node74->node89 + + + + + +node74->node97 + + node75 - -parse_util.cpp - - - -node75->node38 - - + +winsock.h - - -node75->node48 - - + + +node76 + +environment.cpp - - -node79 - -charconv + + +node76->node7 + + - + -node75->node79 - - +node76->node71 + + - - -node76 - -http_client.cpp + + +node78 + +tracer_config.cpp - + -node76->node88 - - +node78->node7 + + + + + +node78->node8 + + - + -node77->node29 - - +node78->node11 + + - - -node78->node56 - - + + +node78->node37 + + - - -node80->node8 - - + + +node78->node49 + + - + + +node78->node71 + + + + -node81->node14 - - +node78->node83 + + - - -node81->node20 - - + + +node78->node104 + + - - -node81->node35 - - + + +node79 + +net_util.cpp - + -node81->node77 - - +node79->node75 + + - - -node82 - -span_matcher.h + + +node99 + +net_util.h - + -node81->node82 - - +node79->node99 + + - - -node82->node14 - - + + +node80 + +parse_util.cpp - + -node82->node35 - - +node80->node39 + + - - -node83 - -version.cpp + + +node80->node49 + + + + + +node84 + +charconv + + + +node80->node84 + + - + + +node81 + +http_client.cpp + + -node83->node80 - - +node81->node92 + + - + -node84->node8 - - +node82->node30 + + - + -node84->node77 - - - - - -node85 - -span_matcher.cpp +node83->node58 + + - + -node85->node7 - - +node85->node8 + + - + + +node86 + +version.cpp + + -node85->node24 - - +node86->node85 + + - - -node85->node38 - - + + +node87 + +span_matcher.h - - -node85->node77 - - + + +node87->node14 + + - + -node85->node82 - - +node87->node36 + + - - -node86 - -tracer.cpp + + +node88->node14 + + - + -node86->node4 - - +node88->node20 + + - - -node86->node7 - - - - + -node86->node9 - - - - - -node86->node11 - - +node88->node36 + + - - -node86->node17 - - + + +node88->node82 + + - - -node86->node24 - - + + +node88->node87 + + - + -node86->node36 - - +node89->node8 + + - - -node86->node40 - - + + +node89->node82 + + - - -node86->node42 - - + + +node90 + +span_matcher.cpp - - -node86->node47 - - + + +node90->node7 + + - - -node86->node48 - - + + +node90->node25 + + - + -node86->node62 - - - - - -node86->node67 - - +node90->node39 + + - - -node86->node68 - - + + +node90->node82 + + - - -node86->node80 - - + + +node90->node87 + + - - -node86->node84 - - + + +node91 + +tracer.cpp - + -node86->node92 - - - - - -node87 - -default_http_client_curl.cpp +node91->node4 + + - - -node87->node30 - - + + +node91->node7 + + - - -node87->node44 - - + + +node91->node9 + + - - -node88->node14 - - + + +node91->node12 + + - - -node88->node35 - - + + +node91->node17 + + - - -node88->node38 - - + + +node91->node25 + + - - -node88->node77 - - + + +node91->node37 + + - - -node89 - -clock.cpp + + +node91->node41 + + - - -node89->node41 - - + + +node91->node43 + + - - -node90 - -collector_response.cpp + + +node91->node48 + + - - -node90->node60 - - + + +node91->node49 + + - - -node91->node26 - - + + +node91->node56 + + - - -node92->node77 - - + + +node91->node64 + + - - -node93 - -limiter.cpp + + +node91->node66 + + - + -node93->node15 - - - - - -node94 - -error.cpp +node91->node71 + + - - -node94->node38 - - + + +node91->node72 + + - + -node96->node14 - - +node91->node85 + + - + + +node91->node89 + + + + + +node91->node99 + + + + -node96->node20 - - +node92->node14 + + - + -node96->node35 - - +node92->node36 + + - + + +node92->node39 + + + + -node96->node77 - - +node92->node82 + + - - -node96->node82 - - + + +node93 + +default_http_client_curl.cpp - - -node97 - -tags.cpp + + +node93->node31 + + - + -node97->node11 - - +node93->node45 + + - - -node97->node48 - - + + +node94 + +clock.cpp - + -node98->node42 - - +node94->node42 + + - - -node99 - -trace_segment.cpp + + +node95->node27 + + - - -node99->node4 - - + + +node96 + +collector_response.cpp - - -node99->node6 - - + + +node96->node63 + + - + -node99->node9 - - +node97->node84 + + - - -node99->node11 - - + + +node98 + +limiter.cpp - + -node99->node24 - - +node98->node15 + + - - -node99->node38 - - + + +node99->node82 + + + + + +node100 + +error.cpp - + -node99->node40 - - +node100->node39 + + - + -node99->node42 - - +node102->node14 + + - - -node99->node56 - - + + +node102->node20 + + - - -node99->node60 - - + + +node102->node36 + + - + -node99->node62 - - - - - -node99->node77 - - +node102->node82 + + - + -node99->node79 - - +node102->node87 + + - - -node100 - -span_sampler_config.cpp + + +node103 + +tags.cpp - - -node100->node7 - - + + +node103->node12 + + - - -node100->node14 - - + + +node103->node49 + + - + -node100->node42 - - +node104->node43 + + - - -node100->node67 - - + + +node105 + +trace_segment.cpp - + -node100->node81 - - +node105->node4 + + + + + +node105->node6 + + + + + +node105->node9 + + + + + +node105->node12 + + + + + +node105->node25 + + + + + +node105->node39 + + + + + +node105->node41 + + + + + +node105->node43 + + + + + +node105->node58 + + + + + +node105->node63 + + + + + +node105->node64 + + + + + +node105->node66 + + + + + +node105->node82 + + + + + +node105->node97 + + + + + +node106 + +span_sampler_config.cpp + + + +node106->node7 + + + + + +node106->node14 + + + + + +node106->node43 + + + + + +node106->node71 + + + + + +node106->node88 + + From b3c9fa454c8beb6270504e1d6d3ad28d94b28c76 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 01:42:36 -0500 Subject: [PATCH 19/29] TODO: AMEND: add a fuzzer for W3C propagation --- CMakeLists.txt | 11 ++++ bin/format | 4 +- fuzz/w3c-propagation/CMakeLists.txt | 3 + fuzz/w3c-propagation/fuzz.cpp | 86 +++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 fuzz/w3c-propagation/CMakeLists.txt create mode 100644 fuzz/w3c-propagation/fuzz.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c93261e..30825d1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ option(BUILD_COVERAGE "Build code with code coverage profiling instrumentation" option(BUILD_EXAMPLE "Build the example program (example/)" OFF) option(BUILD_TESTING "Build the unit tests (test/)" OFF) option(SANITIZE "Build with address sanitizer and undefined behavior sanitizer" OFF) +option(FUZZ_W3C_PROPAGATION "Build a fuzzer for W3C propagation" OFF) set(CMAKE_BUILD_TYPE "RelWithDebInfo") set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -20,6 +21,16 @@ include(ProcessorCount) ProcessorCount(NUM_PROCESSORS) set(MAKE_JOB_COUNT ${NUM_PROCESSORS} CACHE STRING "Number of jobs to use when building libcurl") +if(FUZZ_W3C_PROPAGATION) + add_compile_options(-fsanitize=fuzzer) + add_link_options(-fsanitize=fuzzer) + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) + add_compile_options(-fsanitize=undefined) + add_link_options(-fsanitize=undefined) + add_subdirectory(fuzz/w3c-propagation) +endif() + if (SANITIZE) add_compile_options(-fsanitize=address) add_link_options(-fsanitize=address) diff --git a/bin/format b/bin/format index 5efa9fd9..3c8e97b3 100755 --- a/bin/format +++ b/bin/format @@ -12,7 +12,7 @@ formatter=clang-format-$version formatter_options="--style=file -i $@" find_sources() { - find src/ example/ test/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" + find src/ example/ test/ fuzz/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" } # If the correct version of clang-format is installed, then use it and quit. @@ -35,7 +35,7 @@ if [ "$(docker image ls --quiet $image | wc -l)" -eq 0 ]; then case "$response" in y|Y|yes|Yes|YES|'') ;; *) exit ;; - esac + esac fi mount_path=/mnt/host diff --git a/fuzz/w3c-propagation/CMakeLists.txt b/fuzz/w3c-propagation/CMakeLists.txt new file mode 100644 index 00000000..45c62d40 --- /dev/null +++ b/fuzz/w3c-propagation/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(fuzz fuzz.cpp) + +target_link_libraries(fuzz dd_trace_cpp) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp new file mode 100644 index 00000000..97dcd1fc --- /dev/null +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace dd = datadog::tracing; + +namespace { + +dd::Tracer& tracer_singleton() { + static auto tracer = []() { + dd::TracerConfig config; + config.defaults.service = "fuzzer"; + config.collector = std::make_shared(); + + const auto finalized_config = dd::finalize_config(config); + if (!finalized_config) { + std::abort(); + } + + return dd::Tracer{*finalized_config}; + }(); + + return tracer; +} + +struct MockDictReader : public dd::DictReader { + dd::StringView traceparent; + dd::StringView tracestate; + + dd::Optional lookup(dd::StringView key) const override { + if (key == "traceparent") { + return traceparent; + } + if (key == "tracestate") { + return tracestate; + } + return dd::nullopt; + } + + void visit( + const std::function& + visitor) const override { + visitor("traceparent", traceparent); + visitor("tracestate", tracestate); + } +}; + +struct MockDictWriter : public dd::DictWriter { + void set(dd::StringView, dd::StringView) override {} +}; + +} // namespace + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, size_t size) { + auto& tracer = tracer_singleton(); + + const auto begin_traceparent = reinterpret_cast(data); + const auto end = begin_traceparent + size; + for (const char* begin_tracestate = begin_traceparent; + begin_tracestate != end; ++begin_tracestate) { + MockDictReader reader; + reader.traceparent = dd::range(begin_traceparent, begin_tracestate); + reader.tracestate = dd::range(begin_tracestate, end); + + const auto span = tracer.extract_span(reader); + if (!span) { + continue; + } + + MockDictWriter writer; + span->inject(writer); + } + + return 0; +} From 065945c71e009e03f202db04f427ab15d8e4d815 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 12:04:12 -0500 Subject: [PATCH 20/29] TODO: AMEND: thread_local, not that it matters --- fuzz/w3c-propagation/fuzz.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp index 97dcd1fc..d6262700 100644 --- a/fuzz/w3c-propagation/fuzz.cpp +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -18,7 +18,7 @@ namespace dd = datadog::tracing; namespace { dd::Tracer& tracer_singleton() { - static auto tracer = []() { + thread_local auto tracer = []() { dd::TracerConfig config; config.defaults.service = "fuzzer"; config.collector = std::make_shared(); From 3bbf910ddfb1c142fa5d061e32f9fea116dd102c Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 13:34:10 -0500 Subject: [PATCH 21/29] TODO: AMEND: document the fuzzer --- CMakeLists.txt | 16 +++++++-------- fuzz/README.md | 26 ++++++++++++++++++++++++ fuzz/w3c-propagation/README.md | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 fuzz/README.md create mode 100644 fuzz/w3c-propagation/README.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 30825d1b..0f2442da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,22 +21,22 @@ include(ProcessorCount) ProcessorCount(NUM_PROCESSORS) set(MAKE_JOB_COUNT ${NUM_PROCESSORS} CACHE STRING "Number of jobs to use when building libcurl") -if(FUZZ_W3C_PROPAGATION) - add_compile_options(-fsanitize=fuzzer) - add_link_options(-fsanitize=fuzzer) +function(add_sanitizers) add_compile_options(-fsanitize=address) add_link_options(-fsanitize=address) add_compile_options(-fsanitize=undefined) add_link_options(-fsanitize=undefined) +endfunction() + +if(FUZZ_W3C_PROPAGATION) + add_compile_options(-fsanitize=fuzzer) + add_link_options(-fsanitize=fuzzer) + add_sanitizers() add_subdirectory(fuzz/w3c-propagation) endif() if (SANITIZE) - add_compile_options(-fsanitize=address) - add_link_options(-fsanitize=address) - add_compile_options(-fsanitize=undefined) - add_link_options(-fsanitize=undefined) - # Thread sanitizer fails with "unexpected memory mapping". + add_sanitizers() endif() include (ExternalProject) diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 00000000..3f0181c3 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,26 @@ +Fuzzers +======= +Each subdirectory here contains the source of an executable that [fuzz tests][1] +some part of the library using [LLVM's libfuzzer][2]. + +There is a toplevel CMake boolean option associated with each fuzzer. The naming +convention is `FUZZ_`, e.g. +`FUZZ_W3C_PROPAGATION` for the fuzzer defined in +[fuzz/w3c-propagation/](./w3c-propagation/). The resulting binary is called +`fuzz` by convention. + +When building a fuzzer, the toolchain must be clang-based. For example, this +is how to build the fuzzer in [fuzz/w3c-propagation](./w3c-propagation/) from +the root of the repository: +```console +$ rm -rf .build && mkdir .build # if toolchain or test setup need clearing +$ cd .build +$ CC=clang CXX=clang++ cmake .. -DFUZZ_W3C_PROPAGATION=ON +$ make -j $(nproc) +$ fuzz/w3c-propagation/fuzz + +[... fuzzer output ...] +``` + +[1]: https://en.wikipedia.org/wiki/Fuzzing +[2]: https://llvm.org/docs/LibFuzzer.html diff --git a/fuzz/w3c-propagation/README.md b/fuzz/w3c-propagation/README.md new file mode 100644 index 00000000..ea0edf79 --- /dev/null +++ b/fuzz/w3c-propagation/README.md @@ -0,0 +1,36 @@ +W3C Propagation Fuzzer +====================== +This directory defines an executable, `fuzz`, that fuzz tests extraction and +injection of the W3C tracing HTTP headers "traceparent" and "tracestate". + +Libfuzzer invokes the `LLVMFuzzerTestOneInput` function repeatedly with a binary +blob of varying size and contents. For each blob, [fuzz.cpp](./fuzz.cpp) runs +its test multiple times. The input blob is interpreted in the following way: +```text +blob: _ _ _ _ _ _ _ _ _ _ + +iteration 0: s s s s s s s s s + +iteration 1: p s s s s s s s s + +iteration 2: p p s s s s s s s + +iteration 3: p p p s s s s s s + +... + +iteration 8: p p p p p p p p s + +iteration 9: p p p p p p p p p +``` + The `p`-labeled bytes are the "traceparent" header value, while the `s`-labeled + bytes are the "tracestate" header value. So, for an input blob of length `n`, + the `LLVMFuzzerTestOneInput` runs its test `n + 1` times, for the `n + 1` + possible divisions of the input between "traceparent" and "tracestate". + + Each test uses a singleton[^1] `Tracer` to `extract_span` from a `DictReader` + containing the "traceparent" and "tracestate" headers. If that succeeds, then + the test `inject`s the resulting span into a no-op `DictWriter`. + +[^1]: thread-local, actually, though it doesn't matter because even libfuzzer's + "worker" mode forks instead of threads \ No newline at end of file From dcf6cda4cc34d21f733d49af4826381e6e08896f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 15:21:44 -0500 Subject: [PATCH 22/29] TODO: AMEND: typo in readme --- fuzz/w3c-propagation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz/w3c-propagation/README.md b/fuzz/w3c-propagation/README.md index ea0edf79..f803c278 100644 --- a/fuzz/w3c-propagation/README.md +++ b/fuzz/w3c-propagation/README.md @@ -7,7 +7,7 @@ Libfuzzer invokes the `LLVMFuzzerTestOneInput` function repeatedly with a binary blob of varying size and contents. For each blob, [fuzz.cpp](./fuzz.cpp) runs its test multiple times. The input blob is interpreted in the following way: ```text -blob: _ _ _ _ _ _ _ _ _ _ +blob: _ _ _ _ _ _ _ _ _ iteration 0: s s s s s s s s s @@ -33,4 +33,4 @@ iteration 9: p p p p p p p p p the test `inject`s the resulting span into a no-op `DictWriter`. [^1]: thread-local, actually, though it doesn't matter because even libfuzzer's - "worker" mode forks instead of threads \ No newline at end of file + "worker" mode forks instead of threads From 9a9212c05d2b2bd8a095f7f6f21b034c7cde8ce6 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Thu, 22 Dec 2022 15:29:34 -0500 Subject: [PATCH 23/29] TODO: AMEND: be honest --- fuzz/w3c-propagation/fuzz.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/w3c-propagation/fuzz.cpp b/fuzz/w3c-propagation/fuzz.cpp index d6262700..f02323d6 100644 --- a/fuzz/w3c-propagation/fuzz.cpp +++ b/fuzz/w3c-propagation/fuzz.cpp @@ -68,7 +68,7 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, size_t size) { const auto begin_traceparent = reinterpret_cast(data); const auto end = begin_traceparent + size; for (const char* begin_tracestate = begin_traceparent; - begin_tracestate != end; ++begin_tracestate) { + begin_tracestate <= end; ++begin_tracestate) { MockDictReader reader; reader.traceparent = dd::range(begin_traceparent, begin_tracestate); reader.tracestate = dd::range(begin_tracestate, end); From e923b03087c5d72203d9ee6777f3e7eee48bd691 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 17:46:07 -0500 Subject: [PATCH 24/29] shellcheck --- bin/bazel-build | 2 +- bin/cmake-build | 4 ++-- bin/example | 7 ++++--- bin/format | 8 ++++++-- bin/install-cmake | 2 ++ bin/test | 1 + 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bin/bazel-build b/bin/bazel-build index d46c18e6..4c295c0d 100755 --- a/bin/bazel-build +++ b/bin/bazel-build @@ -2,6 +2,6 @@ set -e -cd $(dirname "$0")/.. +cd "$(dirname "$0")"/.. bazelisk build dd_trace_cpp diff --git a/bin/cmake-build b/bin/cmake-build index 0cd84cf7..1ba61e27 100755 --- a/bin/cmake-build +++ b/bin/cmake-build @@ -2,9 +2,9 @@ set -e -cd $(dirname "$0")/.. +cd "$(dirname "$0")"/.. mkdir -p .build cd .build cmake .. -make -j $(nproc) +make -j "$(nproc)" diff --git a/bin/example b/bin/example index 5139ca41..21dfe2c1 100755 --- a/bin/example +++ b/bin/example @@ -19,8 +19,9 @@ cd "$(dirname "$0")"/.. mkdir -p .build cd .build -cmake .. $coverage_flags -DBUILD_EXAMPLE=1 -make -j $(nproc) $verbosity_flags +cmake .. -DBUILD_EXAMPLE=1 +# shellcheck disable=SC2086 +make -j "$(nproc)" $verbosity_flags if [ "$build_only" -eq 1 ]; then exit @@ -35,6 +36,6 @@ if [ "$DD_API_KEY" = '' ]; then fi docker compose --project-directory ../example up --detach --remove-orphans # docker compose --project-directory ../example logs --follow & -./example/example"$@" +./example/example "$@" docker compose --project-directory ../example down # wait diff --git a/bin/format b/bin/format index 3c8e97b3..588e9230 100755 --- a/bin/format +++ b/bin/format @@ -1,5 +1,7 @@ #!/bin/sh +set -e + # Go to the repository root directory. cd "$(dirname "$0")"/.. @@ -9,7 +11,7 @@ cd "$(dirname "$0")"/.. # occasionally bumps the required version, reformatting everything. version=14 formatter=clang-format-$version -formatter_options="--style=file -i $@" +formatter_options="--style=file -i" find_sources() { find src/ example/ test/ fuzz/ -type f \( -name '*.h' -o -name '*.cpp' \) "$@" @@ -17,6 +19,7 @@ find_sources() { # If the correct version of clang-format is installed, then use it and quit. if >/dev/null command -v "$formatter"; then + # shellcheck disable=SC2086 find_sources -print0 | xargs -0 "$formatter" $formatter_options exit fi @@ -31,7 +34,7 @@ fi image=silkeh/clang:$version if [ "$(docker image ls --quiet $image | wc -l)" -eq 0 ]; then printf '%s is not installed. Download docker image %s instead? [Y/n] ' "$formatter" "$image" - read response + read -r response case "$response" in y|Y|yes|Yes|YES|'') ;; *) exit ;; @@ -52,6 +55,7 @@ process_arg() { } docker_clang_format() { + # shellcheck disable=SC2086 find_sources | while IFS= read -r arg; do process_arg "$arg" done | xargs -0 \ diff --git a/bin/install-cmake b/bin/install-cmake index 66761dcd..26dadba9 100755 --- a/bin/install-cmake +++ b/bin/install-cmake @@ -1,5 +1,7 @@ #!/bin/sh +set -e + # Install a recent binary release of cmake. # Kitware produces a self-installing tarball. diff --git a/bin/test b/bin/test index 79c3afd0..f7cddf38 100755 --- a/bin/test +++ b/bin/test @@ -37,6 +37,7 @@ cd "$(dirname "$0")"/.. mkdir -p .build cd .build +# shellcheck disable=SC2086 cmake .. $coverage_flags -DBUILD_TESTING=1 # Clean up any code coverage artifacts. find . -type f -name '*.gcda' -print0 | xargs -0 rm -f From ac95cd1bc60f66520a3f95646025fb37a18efaf4 Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 17:51:28 -0500 Subject: [PATCH 25/29] shellcheck --- .circleci/config.yml | 9 +++++++++ Dockerfile | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fe7404c..82dd2a21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,14 @@ jobs: - checkout - run: bin/check-format + shellcheck: + docker: + - image: "datadog/docker-library:dd-trace-cpp-ci" + resource_class: small + steps: + - checkout + - run: find bin/ -executable -type f | xargs shellcheck + build-bazel: parameters: toolchain: @@ -59,6 +67,7 @@ workflows: pull-request: jobs: - format + - shellcheck - test-cmake: matrix: parameters: diff --git a/Dockerfile b/Dockerfile index 1e52c44e..32134faa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ env DEBIAN_FRONTEND=noninteractive run apt-get update && apt-get install --yes software-properties-common && \ add-apt-repository ppa:git-core/ppa --yes && \ apt-get update && apt-get upgrade --yes && \ - apt-get install --yes wget build-essential clang sed gdb clang-format git ssh + apt-get install --yes wget build-essential clang sed gdb clang-format git ssh shellcheck # bazelisk, a launcher for bazel. `bazelisk --help` will cause the latest # version to be downloaded. From f789c1c07fe84aeaebd7c460cbb0514dc897750f Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Mon, 9 Jan 2023 18:30:41 -0500 Subject: [PATCH 26/29] tricky rebase... --- bin/test | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/test b/bin/test index f7cddf38..46ca7707 100755 --- a/bin/test +++ b/bin/test @@ -41,6 +41,7 @@ cd .build cmake .. $coverage_flags -DBUILD_TESTING=1 # Clean up any code coverage artifacts. find . -type f -name '*.gcda' -print0 | xargs -0 rm -f +# shellcheck disable=SC2086 make -j "$MAKE_JOB_COUNT" $verbosity_flags if [ "$build_only" -eq 1 ]; then From e36005877bb7e4dd9da31bdef5325a507c939ceb Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Fri, 13 Jan 2023 10:33:16 -0500 Subject: [PATCH 27/29] build unit tests in debug mode --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f2442da..6656675f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,10 @@ if (SANITIZE) add_sanitizers() endif() +if (BUILD_TESTING) + set(CMAKE_BUILD_TYPE "Debug") +endif() + include (ExternalProject) ExternalProject_Add(curl URL "https://github.com/curl/curl/releases/download/curl-7_85_0/curl-7.85.0.tar.gz" From 91ea8a88d55cbbb925c8777ebe98385b2e5d097b Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Fri, 13 Jan 2023 10:39:26 -0500 Subject: [PATCH 28/29] add advisory comments --- src/datadog/trace_segment.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 11040a66..dec56176 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -225,6 +225,7 @@ void TraceSegment::update_decision_maker_trace_tag() { assert(sampling_decision_); + // Note that `found` might be erased below (in case you refactor this code). const auto found = std::find_if( trace_tags_.begin(), trace_tags_.end(), [](const auto& entry) { return entry.first == tags::internal::decision_maker; @@ -237,6 +238,7 @@ void TraceSegment::update_decision_maker_trace_tag() { return; } + // Note that `value` is moved-from below (in case you refactor this code). auto value = "-" + std::to_string(*sampling_decision_->mechanism); if (found == trace_tags_.end()) { trace_tags_.emplace_back(tags::internal::decision_maker, std::move(value)); From 65336af8288b88aad2fbd98d20abebd5ec12a3ba Mon Sep 17 00:00:00 2001 From: David Goffredo Date: Fri, 13 Jan 2023 10:42:25 -0500 Subject: [PATCH 29/29] print on one line, in tests, if empty --- test/mocks/loggers.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/mocks/loggers.cpp b/test/mocks/loggers.cpp index b7c7e962..85a1d308 100644 --- a/test/mocks/loggers.cpp +++ b/test/mocks/loggers.cpp @@ -6,12 +6,16 @@ std::ostream& operator<<(std::ostream& stream, const std::vector& entries) { stream << ""; - for (std::size_t i = 0; i < entries.size(); ++i) { + std::size_t i = 0; + for (; i < entries.size(); ++i) { const auto& entry = entries[i]; const auto kind_name = entry.kind == MockLogger::Entry::ERROR ? "ERROR" : "STARTUP"; stream << '\n' << (i + 1) << ". " << kind_name << ": "; std::visit([&](const auto& value) { stream << value; }, entry.payload); } - return stream << "\n"; + if (i) { + stream << '\n'; + } + return stream << ""; }