WIP Feature/faster ad tls v3B

stan/math/rev/arr/fun/sum.hpp

@@ -29,7 +29,7 @@ class sum_v_vari : public vari {
 
   explicit sum_v_vari(const std::vector<var>& v1)
       : vari(sum_of_val(v1)),
-        v_(reinterpret_cast<vari**>(ChainableStack::instance().memalloc_.alloc(
+        v_(reinterpret_cast<vari**>(ChainableStack::instance_->memalloc_.alloc(
             v1.size() * sizeof(vari*)))),
         length_(v1.size()) {
     for (size_t i = 0; i < length_; i++)

stan/math/rev/core/autodiffstackstorage.hpp

@@ -9,12 +9,10 @@ namespace math {
 
 /**
  * Provides a thread_local singleton if needed. Read warnings below!
- * With STAN_THREADS defined, the singleton is a thread_local static pointer
- * for performance reasons. When STAN_THREADS is not set, we have the old
- * static AD stack in the instance_ field because we saw odd performance
- * issues on the Mac Pro[4]. The rest of this commentary is specifically
- * talking about the design choices in the STAN_THREADS=true case.
- * When a TLS is used then initialization with
+ * For performance reasons the singleton is a global static pointer
+ * which is stored as thread local (TLS) if STAN_THREADS is
+ * defined. The use of a pointer is motivated by performance reasons
+ * for the threading case. When a TLS is used then initialization with
  * a constant expression is required for fast access to the TLS. As
  * the AD storage struct is non-POD it must be initialized as a
  * dynamic expression such that compilers will wrap any access to the
@@ -61,7 +59,6 @@ namespace math {
  * [2] https://github.com/stan-dev/math/pull/826
  * [3]
  * http://discourse.mc-stan.org/t/potentially-dropping-support-for-older-versions-of-apples-version-of-clang/3780/
- * [4] https://github.com/stan-dev/math/pull/1135
  */
 template <typename ChainableT, typename ChainableAllocT>
 struct AutodiffStackSingleton {
@@ -70,12 +67,10 @@ struct AutodiffStackSingleton {
 
   AutodiffStackSingleton() : own_instance_(init()) {}
   ~AutodiffStackSingleton() {
-#ifdef STAN_THREADS
     if (own_instance_) {
       delete instance_;
       instance_ = nullptr;
     }
-#endif
   }
 
   struct AutodiffStackStorage {
@@ -95,39 +90,25 @@ struct AutodiffStackSingleton {
   explicit AutodiffStackSingleton(AutodiffStackSingleton_t const &) = delete;
   AutodiffStackSingleton &operator=(const AutodiffStackSingleton_t &) = delete;
 
-  static constexpr inline AutodiffStackStorage &instance() {
-    return
+  static
 #ifdef STAN_THREADS
-        *
+#ifdef __GNUC__
+      __thread
+#else
+      thread_local
 #endif
-        instance_;
-  }
+#endif
+      AutodiffStackStorage *instance_;
 
  private:
   static bool init() {
-#ifdef STAN_THREADS
     if (!instance_) {
       instance_ = new AutodiffStackStorage();
       return true;
     }
-#endif
     return false;
   }
 
-  static
-#ifdef STAN_THREADS
-#ifdef __GNUC__
-      __thread
-#else
-      thread_local
-#endif
-#endif
-      AutodiffStackStorage
-#ifdef STAN_THREADS
-          *
-#endif
-              instance_;
-
   bool own_instance_;
 };
 
@@ -141,13 +122,8 @@ thread_local
 #endif
     typename AutodiffStackSingleton<ChainableT,
                                     ChainableAllocT>::AutodiffStackStorage
-
-#ifdef STAN_THREADS
         *AutodiffStackSingleton<ChainableT, ChainableAllocT>::instance_
     = nullptr;
-#else
-    AutodiffStackSingleton<ChainableT, ChainableAllocT>::instance_;
-#endif
 
 }  // namespace math
 }  // namespace stan

stan/math/rev/core/build_vari_array.hpp

@@ -20,7 +20,7 @@ namespace math {
 template <int R, int C>
 vari** build_vari_array(const Eigen::Matrix<var, R, C>& x) {
   vari** x_vi_
-      = ChainableStack::instance().memalloc_.alloc_array<vari*>(x.size());
+      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(x.size());
   for (int i = 0; i < x.size(); ++i) {
     x_vi_[i] = x(i).vi_;
   }

stan/math/rev/core/chainable_alloc.hpp

@@ -16,7 +16,7 @@ namespace math {
 class chainable_alloc {
  public:
   chainable_alloc() {
-    ChainableStack::instance().var_alloc_stack_.push_back(this);
+    ChainableStack::instance_->var_alloc_stack_.push_back(this);
   }
   virtual ~chainable_alloc() {}
 };

stan/math/rev/core/empty_nested.hpp

@@ -10,7 +10,7 @@ namespace math {
  * Return true if there is no nested autodiff being executed.
  */
 static inline bool empty_nested() {
-  return ChainableStack::instance().nested_var_stack_sizes_.empty();
+  return ChainableStack::instance_->nested_var_stack_sizes_.empty();
 }
 
 }  // namespace math

stan/math/rev/core/gevv_vvv_vari.hpp

@@ -32,7 +32,7 @@ class gevv_vvv_vari : public vari {
         length_(length) {
     alpha_ = alpha->vi_;
     // TODO(carpenter): replace this with array alloc fun call
-    v1_ = reinterpret_cast<vari**>(ChainableStack::instance().memalloc_.alloc(
+    v1_ = reinterpret_cast<vari**>(ChainableStack::instance_->memalloc_.alloc(
         2 * length_ * sizeof(vari*)));
     v2_ = v1_ + length_;
     for (size_t i = 0; i < length_; i++)

stan/math/rev/core/grad.hpp

@@ -37,8 +37,8 @@ static void grad(vari* vi) {
 
   typedef std::vector<vari*>::reverse_iterator it_t;
   vi->init_dependent();
-  it_t begin = ChainableStack::instance().var_stack_.rbegin();
-  it_t end = empty_nested() ? ChainableStack::instance().var_stack_.rend()
+  it_t begin = ChainableStack::instance_->var_stack_.rbegin();
+  it_t end = empty_nested() ? ChainableStack::instance_->var_stack_.rend()
                             : begin + nested_size();
   for (it_t it = begin; it < end; ++it) {
     (*it)->chain();

stan/math/rev/core/nested_size.hpp

@@ -8,8 +8,8 @@ namespace stan {
 namespace math {
 
 static inline size_t nested_size() {
-  return ChainableStack::instance().var_stack_.size()
-         - ChainableStack::instance().nested_var_stack_sizes_.back();
+  return ChainableStack::instance_->var_stack_.size()
+         - ChainableStack::instance_->nested_var_stack_sizes_.back();
 }
 
 }  // namespace math

stan/math/rev/core/precomputed_gradients.hpp

@@ -52,9 +52,9 @@ class precomputed_gradients_vari : public vari {
                              const std::vector<double>& gradients)
       : vari(val),
         size_(vars.size()),
-        varis_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             vars.size())),
-        gradients_(ChainableStack::instance().memalloc_.alloc_array<double>(
+        gradients_(ChainableStack::instance_->memalloc_.alloc_array<double>(
             vars.size())) {
     check_consistent_sizes("precomputed_gradients_vari", "vars", vars,
                            "gradients", gradients);

stan/math/rev/core/print_stack.hpp

@@ -18,14 +18,14 @@ namespace math {
  * @param o ostream to modify
  */
 inline void print_stack(std::ostream& o) {
-  o << "STACK, size=" << ChainableStack::instance().var_stack_.size()
+  o << "STACK, size=" << ChainableStack::instance_->var_stack_.size()
     << std::endl;
   // TODO(carpenter): this shouldn't need to be cast any more
-  for (size_t i = 0; i < ChainableStack::instance().var_stack_.size(); ++i)
-    o << i << "  " << ChainableStack::instance().var_stack_[i] << "  "
-      << (static_cast<vari*>(ChainableStack::instance().var_stack_[i]))->val_
+  for (size_t i = 0; i < ChainableStack::instance_->var_stack_.size(); ++i)
+    o << i << "  " << ChainableStack::instance_->var_stack_[i] << "  "
+      << (static_cast<vari*>(ChainableStack::instance_->var_stack_[i]))->val_
       << " : "
-      << (static_cast<vari*>(ChainableStack::instance().var_stack_[i]))->adj_
+      << (static_cast<vari*>(ChainableStack::instance_->var_stack_[i]))->adj_
       << std::endl;
 }
 

stan/math/rev/core/recover_memory.hpp

@@ -20,13 +20,13 @@ static inline void recover_memory() {
     throw std::logic_error(
         "empty_nested() must be true"
         " before calling recover_memory()");
-  ChainableStack::instance().var_stack_.clear();
-  ChainableStack::instance().var_nochain_stack_.clear();
-  for (auto &x : ChainableStack::instance().var_alloc_stack_) {
+  ChainableStack::instance_->var_stack_.clear();
+  ChainableStack::instance_->var_nochain_stack_.clear();
+  for (auto &x : ChainableStack::instance_->var_alloc_stack_) {
     delete x;
   }
-  ChainableStack::instance().var_alloc_stack_.clear();
-  ChainableStack::instance().memalloc_.recover_all();
+  ChainableStack::instance_->var_alloc_stack_.clear();
+  ChainableStack::instance_->memalloc_.recover_all();
 }
 
 }  // namespace math

stan/math/rev/core/recover_memory_nested.hpp

@@ -23,24 +23,24 @@ static inline void recover_memory_nested() {
         "empty_nested() must be false"
         " before calling recover_memory_nested()");
 
-  ChainableStack::instance().var_stack_.resize(
-      ChainableStack::instance().nested_var_stack_sizes_.back());
-  ChainableStack::instance().nested_var_stack_sizes_.pop_back();
+  ChainableStack::instance_->var_stack_.resize(
+      ChainableStack::instance_->nested_var_stack_sizes_.back());
+  ChainableStack::instance_->nested_var_stack_sizes_.pop_back();
 
-  ChainableStack::instance().var_nochain_stack_.resize(
-      ChainableStack::instance().nested_var_nochain_stack_sizes_.back());
-  ChainableStack::instance().nested_var_nochain_stack_sizes_.pop_back();
+  ChainableStack::instance_->var_nochain_stack_.resize(
+      ChainableStack::instance_->nested_var_nochain_stack_sizes_.back());
+  ChainableStack::instance_->nested_var_nochain_stack_sizes_.pop_back();
 
   for (size_t i
-       = ChainableStack::instance().nested_var_alloc_stack_starts_.back();
-       i < ChainableStack::instance().var_alloc_stack_.size(); ++i) {
-    delete ChainableStack::instance().var_alloc_stack_[i];
+       = ChainableStack::instance_->nested_var_alloc_stack_starts_.back();
+       i < ChainableStack::instance_->var_alloc_stack_.size(); ++i) {
+    delete ChainableStack::instance_->var_alloc_stack_[i];
   }
-  ChainableStack::instance().var_alloc_stack_.resize(
-      ChainableStack::instance().nested_var_alloc_stack_starts_.back());
-  ChainableStack::instance().nested_var_alloc_stack_starts_.pop_back();
+  ChainableStack::instance_->var_alloc_stack_.resize(
+      ChainableStack::instance_->nested_var_alloc_stack_starts_.back());
+  ChainableStack::instance_->nested_var_alloc_stack_starts_.pop_back();
 
-  ChainableStack::instance().memalloc_.recover_nested();
+  ChainableStack::instance_->memalloc_.recover_nested();
 }
 
 }  // namespace math

stan/math/rev/core/set_zero_all_adjoints.hpp

@@ -12,9 +12,9 @@ namespace math {
  * Reset all adjoint values in the stack to zero.
  */
 static void set_zero_all_adjoints() {
-  for (auto &x : ChainableStack::instance().var_stack_)
+  for (auto &x : ChainableStack::instance_->var_stack_)
     x->set_zero_adjoint();
-  for (auto &x : ChainableStack::instance().var_nochain_stack_)
+  for (auto &x : ChainableStack::instance_->var_nochain_stack_)
     x->set_zero_adjoint();
 }
 

stan/math/rev/core/set_zero_all_adjoints_nested.hpp

@@ -19,17 +19,17 @@ static void set_zero_all_adjoints_nested() {
     throw std::logic_error(
         "empty_nested() must be false before calling"
         " set_zero_all_adjoints_nested()");
-  size_t start1 = ChainableStack::instance().nested_var_stack_sizes_.back();
+  size_t start1 = ChainableStack::instance_->nested_var_stack_sizes_.back();
   // avoid wrap with unsigned when start1 == 0
   for (size_t i = (start1 == 0U) ? 0U : (start1 - 1);
-       i < ChainableStack::instance().var_stack_.size(); ++i)
-    ChainableStack::instance().var_stack_[i]->set_zero_adjoint();
+       i < ChainableStack::instance_->var_stack_.size(); ++i)
+    ChainableStack::instance_->var_stack_[i]->set_zero_adjoint();
 
   size_t start2
-      = ChainableStack::instance().nested_var_nochain_stack_sizes_.back();
+      = ChainableStack::instance_->nested_var_nochain_stack_sizes_.back();
   for (size_t i = (start2 == 0U) ? 0U : (start2 - 1);
-       i < ChainableStack::instance().var_nochain_stack_.size(); ++i) {
-    ChainableStack::instance().var_nochain_stack_[i]->set_zero_adjoint();
+       i < ChainableStack::instance_->var_nochain_stack_.size(); ++i) {
+    ChainableStack::instance_->var_nochain_stack_[i]->set_zero_adjoint();
   }
 }
 

stan/math/rev/core/start_nested.hpp

@@ -11,13 +11,13 @@ namespace math {
  * can find it.
  */
 static inline void start_nested() {
-  ChainableStack::instance().nested_var_stack_sizes_.push_back(
-      ChainableStack::instance().var_stack_.size());
-  ChainableStack::instance().nested_var_nochain_stack_sizes_.push_back(
-      ChainableStack::instance().var_nochain_stack_.size());
-  ChainableStack::instance().nested_var_alloc_stack_starts_.push_back(
-      ChainableStack::instance().var_alloc_stack_.size());
-  ChainableStack::instance().memalloc_.start_nested();
+  ChainableStack::instance_->nested_var_stack_sizes_.push_back(
+      ChainableStack::instance_->var_stack_.size());
+  ChainableStack::instance_->nested_var_nochain_stack_sizes_.push_back(
+      ChainableStack::instance_->var_nochain_stack_.size());
+  ChainableStack::instance_->nested_var_alloc_stack_starts_.push_back(
+      ChainableStack::instance_->var_alloc_stack_.size());
+  ChainableStack::instance_->memalloc_.start_nested();
 }
 
 }  // namespace math

stan/math/rev/core/vari.hpp

@@ -56,14 +56,14 @@ class vari {
    * @param x Value of the constructed variable.
    */
   explicit vari(double x) : val_(x), adj_(0.0) {
-    ChainableStack::instance().var_stack_.push_back(this);
+    ChainableStack::instance_->var_stack_.push_back(this);
   }
 
   vari(double x, bool stacked) : val_(x), adj_(0.0) {
     if (stacked)
-      ChainableStack::instance().var_stack_.push_back(this);
+      ChainableStack::instance_->var_stack_.push_back(this);
     else
-      ChainableStack::instance().var_nochain_stack_.push_back(this);
+      ChainableStack::instance_->var_nochain_stack_.push_back(this);
   }
 
   /**
@@ -123,7 +123,7 @@ class vari {
    * @return Pointer to allocated bytes.
    */
   static inline void* operator new(size_t nbytes) {
-    return ChainableStack::instance().memalloc_.alloc(nbytes);
+    return ChainableStack::instance_->memalloc_.alloc(nbytes);
   }
 
   /**

stan/math/rev/mat/fun/cholesky_decompose.hpp

@@ -79,9 +79,9 @@ class cholesky_block : public vari {
                  const Eigen::Matrix<double, -1, -1>& L_A)
       : vari(0.0),
         M_(A.rows()),
-        vari_ref_A_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_A_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)),
-        vari_ref_L_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_L_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)) {
     size_t pos = 0;
     block_size_ = std::max(M_ / 8, 8);
@@ -191,9 +191,9 @@ class cholesky_scalar : public vari {
                   const Eigen::Matrix<double, -1, -1>& L_A)
       : vari(0.0),
         M_(A.rows()),
-        vari_ref_A_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_A_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)),
-        vari_ref_L_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_L_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)) {
     size_t accum = 0;
     size_t accum_i = accum;
@@ -281,9 +281,9 @@ class cholesky_opencl : public vari {
                   const Eigen::Matrix<double, -1, -1>& L_A)
       : vari(0.0),
         M_(A.rows()),
-        vari_ref_A_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_A_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)),
-        vari_ref_L_(ChainableStack::instance().memalloc_.alloc_array<vari*>(
+        vari_ref_L_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             A.rows() * (A.rows() + 1) / 2)) {
     size_t pos = 0;
     for (size_type j = 0; j < M_; ++j) {