128-bit trace IDs

.circleci/config.yml

@@ -16,7 +16,7 @@ jobs:
     resource_class: small
     steps:
     - checkout
-    - run: find bin/ -executable -type f | xargs shellcheck
+    - run: find bin/ -executable -type f -print0 | xargs -0 shellcheck
 
   build-bazel:
     parameters:

BUILD.bazel

@@ -47,6 +47,7 @@ cc_library(
     "src/datadog/threaded_event_scheduler.cpp",
     "src/datadog/tracer_config.cpp",
     "src/datadog/tracer.cpp",
+    "src/datadog/trace_id.cpp",
     "src/datadog/trace_sampler_config.cpp",
     "src/datadog/trace_sampler.cpp",
     "src/datadog/trace_segment.cpp",
@@ -101,6 +102,7 @@ cc_library(
     "src/datadog/threaded_event_scheduler.h",
     "src/datadog/tracer_config.h",
     "src/datadog/tracer.h",
+    "src/datadog/trace_id.h",
     "src/datadog/trace_sampler_config.h",
     "src/datadog/trace_sampler.h",
     "src/datadog/trace_segment.h",

CMakeLists.txt

@@ -116,6 +116,7 @@ target_sources(dd_trace_cpp PRIVATE
     src/datadog/threaded_event_scheduler.cpp
     src/datadog/tracer_config.cpp
     src/datadog/tracer.cpp
+    src/datadog/trace_id.cpp
     src/datadog/trace_sampler_config.cpp
     src/datadog/trace_sampler.cpp
     src/datadog/trace_segment.cpp
@@ -176,6 +177,7 @@ target_sources(dd_trace_cpp PUBLIC
   src/datadog/threaded_event_scheduler.h
   src/datadog/tracer_config.h
   src/datadog/tracer.h
+  src/datadog/trace_id.h
   src/datadog/trace_sampler_config.h
   src/datadog/trace_sampler.h
   src/datadog/trace_segment.h

bin/check

@@ -11,3 +11,4 @@ cd "$(dirname "$0")"/..
 bin/format --dry-run -Werror
 bin/test
 bin/bazel-build
+find bin/ -executable -type f -print0 | xargs -0 shellcheck

src/datadog/environment.h

@@ -46,7 +46,8 @@ namespace environment {
   MACRO(DD_TRACE_SAMPLING_RULES)              \
   MACRO(DD_TRACE_STARTUP_LOGS)                \
   MACRO(DD_TRACE_TAGS_PROPAGATION_MAX_LENGTH) \
-  MACRO(DD_VERSION)
+  MACRO(DD_VERSION)                           \
+  MACRO(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
 
 #define WITH_COMMA(ARG) ARG,
 

src/datadog/extracted_data.h

@@ -9,22 +9,17 @@
 #include <vector>
 
 #include "optional.h"
+#include "trace_id.h"
 
 namespace datadog {
 namespace tracing {
 
 struct ExtractedData {
-  Optional<std::uint64_t> trace_id;
+  Optional<TraceID> trace_id;
   Optional<std::uint64_t> parent_id;
   Optional<std::string> origin;
   std::vector<std::pair<std::string, std::string>> trace_tags;
   Optional<int> sampling_priority;
-  // If this `ExtractedData` was created on account of `PropagationStyle::W3C`
-  // and trace context was successfully extracted, then `full_w3c_trace_id_hex`
-  // contains the hex-encoded 128-bit trace ID. `trace_id` will be the least
-  // significant 64 bits of the same value. `full_w3c_trace_id_hex` is used for
-  // the `W3C` injection style.
-  Optional<std::string> full_w3c_trace_id_hex;
   // If this `ExtractedData` was created on account of `PropagationStyle::W3C`,
   // then `additional_w3c_tracestate` contains the parts of the "tracestate"
   // header that are not the "dd" (Datadog) entry. If there are no other parts,

src/datadog/hex.h

@@ -1,6 +1,6 @@
 #pragma once
 
-// This component provides a function, `hex`, for formatting an integral value
+// This component provides functions for formatting an unsigned integral value
 // in hexadecimal.
 
 #include <cassert>
@@ -12,12 +12,14 @@
 namespace datadog {
 namespace tracing {
 
-// Return the specified `value` formatted as a lower-case hexadecimal string
-// without any leading zeroes.
-template <typename Integer>
-std::string hex(Integer value) {
-  // 4 bits per hex digit char, and then +1 char for possible minus sign
-  char buffer[std::numeric_limits<Integer>::digits / 4 + 1];
+// Return the specified unsigned `value` formatted as a lower-case hexadecimal
+// string without any leading zeroes.
+template <typename UnsignedInteger>
+std::string hex(UnsignedInteger value) {
+  static_assert(!std::numeric_limits<UnsignedInteger>::is_signed);
+
+  // 4 bits per hex digit char
+  char buffer[std::numeric_limits<UnsignedInteger>::digits / 4];
 
   const int base = 16;
   auto result =
@@ -27,5 +29,25 @@ std::string hex(Integer value) {
   return std::string{std::begin(buffer), result.ptr};
 }
 
+// Return the specified unsigned `value` formatted as a lower-case hexadecimal
+// string with leading zeroes.
+template <typename UnsignedInteger>
+std::string hex_padded(UnsignedInteger value) {
+  static_assert(!std::numeric_limits<UnsignedInteger>::is_signed);
+
+  // 4 bits per hex digit char.
+  char buffer[std::numeric_limits<UnsignedInteger>::digits / 4];
+
+  const int base = 16;
+  auto result =
+      std::to_chars(std::begin(buffer), std::end(buffer), value, base);
+  assert(result.ec == std::errc());
+
+  const auto num_zeroes = sizeof(buffer) - (result.ptr - std::begin(buffer));
+  std::string padded(num_zeroes, '0');
+  padded.append(std::begin(buffer), result.ptr);
+  return padded;
+}
+
 }  // namespace tracing
 }  // namespace datadog

src/datadog/id_generator.cpp

@@ -16,7 +16,7 @@ namespace {
 extern "C" void on_fork();
 #endif
 
-class DefaultIDGenerator {
+class Uint64Generator {
   std::mt19937_64 generator_;
   // The distribution used is for a _signed_ integer, and the default minimum
   // value is zero.
@@ -26,7 +26,7 @@ class DefaultIDGenerator {
   std::uniform_int_distribution<std::int64_t> distribution_;
 
  public:
-  DefaultIDGenerator() {
+  Uint64Generator() {
     seed_with_random();
 // If a process links to this library and then calls `fork`, the `generator_` in
 // the parent and child processes will produce the exact same sequence of
@@ -48,17 +48,36 @@ class DefaultIDGenerator {
   void seed_with_random() { generator_.seed(std::random_device{}()); }
 };
 
-thread_local DefaultIDGenerator thread_local_generator;
+thread_local Uint64Generator thread_local_generator;
 
 #ifndef _MSC_VER
 void on_fork() { thread_local_generator.seed_with_random(); }
 #endif
 
-}  // namespace
+class DefaultIDGenerator : public IDGenerator {
+  const bool trace_id_128_bit_;
+
+ public:
+  explicit DefaultIDGenerator(bool trace_id_128_bit)
+      : trace_id_128_bit_(trace_id_128_bit) {}
+
+  TraceID trace_id() const override {
+    TraceID result;
+    result.low = thread_local_generator();
+    if (trace_id_128_bit_) {
+      result.high = thread_local_generator();
+    }
+    return result;
+  }
 
-const IDGenerator default_id_generator = []() {
-  return thread_local_generator();
+  std::uint64_t span_id() const override { return thread_local_generator(); }
 };
 
+}  // namespace
+
+std::shared_ptr<const IDGenerator> default_id_generator(bool trace_id_128_bit) {
+  return std::make_shared<DefaultIDGenerator>(trace_id_128_bit);
+}
+
 }  // namespace tracing
 }  // namespace datadog

src/datadog/id_generator.h

@@ -3,23 +3,29 @@
 // This component provides facilities for generating sequences of IDs used as
 // span IDs and trace IDs.
 //
-// `IDGenerator` is an alias for `std::function<std::uint64_t()>`.
+// `IDGenerator` is an interface for generating trace IDs and span IDs.
 //
 // `default_id_generator` is an `IDGenerator` that produces a thread-local
 // pseudo-random sequence of uniformly distributed 63-bit unsigned integers. The
-// sequence is randomly seeded one per thread and anytime the process forks.
-// The IDs are 63-bit (instead of 64-bit) to ease compatibility with peer
-// runtimes that don't have a native 64-bit unsigned numeric type.
+// sequence is randomly seeded once per thread and anytime the process forks.
 
 #include <cstdint>
-#include <functional>
+#include <memory>
+
+#include "trace_id.h"
 
 namespace datadog {
 namespace tracing {
 
-using IDGenerator = std::function<std::uint64_t()>;
+class IDGenerator {
+ public:
+  virtual ~IDGenerator() = default;
+
+  virtual std::uint64_t span_id() const = 0;
+  virtual TraceID trace_id() const = 0;
+};
 
-extern const IDGenerator default_id_generator;
+std::shared_ptr<const IDGenerator> default_id_generator(bool trace_id_128_bit);
 
 }  // namespace tracing
 }  // namespace datadog

src/datadog/limiter.cpp

@@ -2,7 +2,6 @@
 
 #include <algorithm>
 #include <cmath>
-#include <iostream>
 #include <numeric>
 
 namespace datadog {

src/datadog/sampling_util.h

@@ -11,6 +11,16 @@
 namespace datadog {
 namespace tracing {
 
+// Return a hash value for the specified `value`.  `value` is one of the
+// following:
+//
+// - a 64-bit span ID
+// - a 64-bit trace ID
+// - the lower 64 bits of a 128-bit trace ID
+//
+// The resulting hash value is compared with an upper bound provided by
+// `max_id_from_rate` (below) to determine whether the span/trace associated
+// with `value` is eligible for keeping on statistical grounds.
 inline std::uint64_t knuth_hash(std::uint64_t value) {
   return value * UINT64_C(1111111111111111111);
 }

src/datadog/span.cpp

@@ -15,7 +15,8 @@ namespace datadog {
 namespace tracing {
 
 Span::Span(SpanData* data, const std::shared_ptr<TraceSegment>& trace_segment,
-           const IDGenerator& generate_span_id, const Clock& clock)
+           const std::function<std::uint64_t()>& generate_span_id,
+           const Clock& clock)
     : trace_segment_(trace_segment),
       data_(data),
       generate_span_id_(generate_span_id),
@@ -62,7 +63,7 @@ void Span::inject(DictWriter& writer) const {
 
 std::uint64_t Span::id() const { return data_->span_id; }
 
-std::uint64_t Span::trace_id() const { return data_->trace_id; }
+TraceID Span::trace_id() const { return data_->trace_id; }
 
 Optional<std::uint64_t> Span::parent_id() const {
   if (data_->parent_id == 0) {

src/datadog/span.h

@@ -41,14 +41,15 @@
 // via the `set_end_time` member function prior to the span's destruction.
 
 #include <chrono>
+#include <cstdint>
 #include <functional>
 #include <memory>
 
 #include "clock.h"
 #include "error.h"
-#include "id_generator.h"
 #include "optional.h"
 #include "string_view.h"
+#include "trace_id.h"
 
 namespace datadog {
 namespace tracing {
@@ -61,19 +62,18 @@ class TraceSegment;
 class Span {
   std::shared_ptr<TraceSegment> trace_segment_;
   SpanData* data_;
-  IDGenerator generate_span_id_;
+  std::function<std::uint64_t()> generate_span_id_;
   Clock clock_;
   Optional<std::chrono::steady_clock::time_point> end_time_;
 
  public:
-  // Create a span whose properties are stored in the specified `data` and that
-  // is associated with the specified `trace_segment`.  Optionally specify
-  // `generate_span_id` to generate IDs of child spans, and a `clock` to
-  // determine start and end times.  If `generate_span_id` and `clock` are not
-  // specified`, then `default_id_generator` and `default_clock` are used
-  // instead respectively.
+  // Create a span whose properties are stored in the specified `data`, that is
+  // associated with the specified `trace_segment`, that uses the specified
+  // `generate_span_id` to generate IDs of child spans, and that uses the
+  // specified `clock` to determine start and end times.
   Span(SpanData* data, const std::shared_ptr<TraceSegment>& trace_segment,
-       const IDGenerator& generate_span_id, const Clock& clock);
+       const std::function<std::uint64_t()>& generate_span_id,
+       const Clock& clock);
   Span(const Span&) = delete;
   Span(Span&&) = default;
   Span& operator=(Span&&) = default;
@@ -95,7 +95,7 @@ class Span {
   // Return this span's ID (span ID).
   std::uint64_t id() const;
   // Return the ID of the trace of which this span is a part.
-  std::uint64_t trace_id() const;
+  TraceID trace_id() const;
   // Return the ID of this span's parent span, or return null if this span has
   // no parent.
   Optional<std::uint64_t> parent_id() const;

src/datadog/span_data.cpp

@@ -78,7 +78,7 @@ Expected<void> msgpack_encode(std::string& destination, const SpanData& span) {
          return msgpack::pack_string(destination, span.resource);
        },
       "trace_id", [&](auto& destination) {
-         msgpack::pack_integer(destination, span.trace_id);
+         msgpack::pack_integer(destination, span.trace_id.low);
          return Expected<void>{};
        },
       "span_id", [&](auto& destination) {

src/datadog/span_data.h

@@ -12,6 +12,7 @@
 #include "expected.h"
 #include "optional.h"
 #include "string_view.h"
+#include "trace_id.h"
 
 namespace datadog {
 namespace tracing {
@@ -24,7 +25,7 @@ struct SpanData {
   std::string service_type;
   std::string name;
   std::string resource;
-  std::uint64_t trace_id = 0;
+  TraceID trace_id;
   std::uint64_t span_id = 0;
   std::uint64_t parent_id = 0;
   TimePoint start;

src/datadog/tags.cpp

@@ -29,6 +29,7 @@ const std::string span_sampling_mechanism = "_dd.span_sampling.mechanism";
 const std::string span_sampling_rule_rate = "_dd.span_sampling.rule_rate";
 const std::string span_sampling_limit = "_dd.span_sampling.max_per_second";
 const std::string w3c_extraction_error = "_dd.w3c_extraction_error";
+const std::string trace_id_high = "_dd.p.tid";
 
 }  // namespace internal
 

src/datadog/tags.h

@@ -33,6 +33,7 @@ extern const std::string span_sampling_mechanism;
 extern const std::string span_sampling_rule_rate;
 extern const std::string span_sampling_limit;
 extern const std::string w3c_extraction_error;
+extern const std::string trace_id_high;
 }  // namespace internal
 
 // Return whether the specified `tag_name` is reserved for use internal to this