diff --git a/inst/include/tbb_local/serial/tbb/parallel_for.h b/inst/include/tbb_local/serial/tbb/parallel_for.h index e5959c22..5b42a670 100644 --- a/inst/include/tbb_local/serial/tbb/parallel_for.h +++ b/inst/include/tbb_local/serial/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2018 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_SERIAL_parallel_for_H diff --git a/inst/include/tbb_local/serial/tbb/tbb_annotate.h b/inst/include/tbb_local/serial/tbb/tbb_annotate.h index c16defea..6b79be2d 100644 --- a/inst/include/tbb_local/serial/tbb/tbb_annotate.h +++ b/inst/include/tbb_local/serial/tbb/tbb_annotate.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2018 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_annotate_H diff --git a/inst/include/tbb_local/tbb/aggregator.h b/inst/include/tbb_local/tbb/aggregator.h index 6aecbb74..33e83679 100644 --- a/inst/include/tbb_local/tbb/aggregator.h +++ b/inst/include/tbb_local/tbb/aggregator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__aggregator_H diff --git a/inst/include/tbb_local/tbb/aligned_space.h b/inst/include/tbb_local/tbb/aligned_space.h index 56fd85f3..03b09d09 100644 --- a/inst/include/tbb_local/tbb/aligned_space.h +++ b/inst/include/tbb_local/tbb/aligned_space.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_aligned_space_H @@ -36,10 +32,10 @@ class aligned_space { element_type array[(sizeof(T)*N+sizeof(element_type)-1)/sizeof(element_type)]; public: //! Pointer to beginning of array - T* begin() {return internal::punned_cast(this);} + T* begin() const {return internal::punned_cast(this);} //! Pointer to one past last element in array. - T* end() {return begin()+N;} + T* end() const {return begin()+N;} }; } // namespace tbb diff --git a/inst/include/tbb_local/tbb/atomic.h b/inst/include/tbb_local/tbb/atomic.h index 72ec534e..92d1e11a 100644 --- a/inst/include/tbb_local/tbb/atomic.h +++ b/inst/include/tbb_local/tbb/atomic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_atomic_H diff --git a/inst/include/tbb_local/tbb/blocked_range.h b/inst/include/tbb_local/tbb/blocked_range.h index 9f24cd2b..d1ff1f45 100644 --- a/inst/include/tbb_local/tbb/blocked_range.h +++ b/inst/include/tbb_local/tbb/blocked_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range_H @@ -25,6 +21,15 @@ namespace tbb { +namespace internal { + +// blocked_rangeNd_impl forward declaration in tbb::internal namespace to +// name it as a friend for a tbb::blocked_range. +template +class blocked_rangeNd_impl; + +} // namespace internal + /** \page range_req Requirements on range concept Class \c R implementing the concept of range must define: - \code R::R( const R& ); \endcode Copy constructor @@ -47,9 +52,11 @@ class blocked_range { //! Type for size of a range typedef std::size_t size_type; - //! Construct range with default-constructed values for begin and end. +#if __TBB_DEPRECATED_BLOCKED_RANGE_DEFAULT_CTOR + //! Construct range with default-constructed values for begin, end, and grainsize. /** Requires that Value have a default constructor. */ - blocked_range() : my_end(), my_begin() {} + blocked_range() : my_end(), my_begin(), my_grainsize() {} +#endif //! Construct range over half-open interval [begin,end), with the given grainsize. blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : @@ -115,13 +122,12 @@ class blocked_range { #endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ private: - /** NOTE: my_end MUST be declared before my_begin, otherwise the forking constructor will break. */ + /** NOTE: my_end MUST be declared before my_begin, otherwise the splitting constructor will break. */ Value my_end; Value my_begin; size_type my_grainsize; - //! Auxiliary function used by forking constructor. - /** Using this function lets us not require that Value support assignment or default construction. */ + //! Auxiliary function used by the splitting constructor. static Value do_split( blocked_range& r, split ) { __TBB_ASSERT( r.is_divisible(), "cannot split blocked_range that is not divisible" ); @@ -152,6 +158,9 @@ class blocked_range { template friend class blocked_range3d; + + template + friend class internal::blocked_rangeNd_impl; }; } // namespace tbb diff --git a/inst/include/tbb_local/tbb/blocked_range2d.h b/inst/include/tbb_local/tbb/blocked_range2d.h index f1b9f35d..cd0fe1c3 100644 --- a/inst/include/tbb_local/tbb/blocked_range2d.h +++ b/inst/include/tbb_local/tbb/blocked_range2d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range2d_H @@ -45,19 +41,17 @@ class blocked_range2d { ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : my_rows(row_begin,row_end,row_grainsize), my_cols(col_begin,col_end,col_grainsize) - { - } + {} blocked_range2d( RowValue row_begin, RowValue row_end, ColValue col_begin, ColValue col_end ) : my_rows(row_begin,row_end), my_cols(col_begin,col_end) - { - } + {} //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_rows.empty() || my_cols.empty(); } @@ -86,6 +80,14 @@ class blocked_range2d { } #endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ + //! The rows of the iteration space + const row_range_type& rows() const {return my_rows;} + + //! The columns of the iteration space + const col_range_type& cols() const {return my_cols;} + +private: + template void do_split( blocked_range2d& r, Split& split_obj ) { @@ -95,12 +97,6 @@ class blocked_range2d { my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); } } - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} }; } // namespace tbb diff --git a/inst/include/tbb_local/tbb/blocked_range3d.h b/inst/include/tbb_local/tbb/blocked_range3d.h index c62565ee..5c6cf9f0 100644 --- a/inst/include/tbb_local/tbb/blocked_range3d.h +++ b/inst/include/tbb_local/tbb/blocked_range3d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range3d_H @@ -49,8 +45,7 @@ class blocked_range3d { my_pages(page_begin,page_end), my_rows(row_begin,row_end), my_cols(col_begin,col_end) - { - } + {} blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize, RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, @@ -58,12 +53,11 @@ class blocked_range3d { my_pages(page_begin,page_end,page_grainsize), my_rows(row_begin,row_end,row_grainsize), my_cols(col_begin,col_end,col_grainsize) - { - } + {} //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_pages.empty() || my_rows.empty() || my_cols.empty(); } @@ -94,6 +88,17 @@ class blocked_range3d { } #endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ + //! The pages of the iteration space + const page_range_type& pages() const {return my_pages;} + + //! The rows of the iteration space + const row_range_type& rows() const {return my_rows;} + + //! The columns of the iteration space + const col_range_type& cols() const {return my_cols;} + +private: + template void do_split( blocked_range3d& r, Split& split_obj) { @@ -103,7 +108,7 @@ class blocked_range3d { } else { my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); } - } else { + } else { if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) { my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); } else { @@ -111,16 +116,6 @@ class blocked_range3d { } } } - - //! The pages of the iteration space - const page_range_type& pages() const {return my_pages;} - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - }; } // namespace tbb diff --git a/inst/include/tbb_local/tbb/blocked_rangeNd.h b/inst/include/tbb_local/tbb/blocked_rangeNd.h new file mode 100644 index 00000000..b623d002 --- /dev/null +++ b/inst/include/tbb_local/tbb/blocked_rangeNd.h @@ -0,0 +1,150 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_rangeNd_H +#define __TBB_blocked_rangeNd_H + +#if ! TBB_PREVIEW_BLOCKED_RANGE_ND + #error Set TBB_PREVIEW_BLOCKED_RANGE_ND to include blocked_rangeNd.h +#endif + +#include "tbb_config.h" + +// tbb::blocked_rangeNd requires C++11 support +#if __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +#include "internal/_template_helpers.h" // index_sequence, make_index_sequence + +#include +#include // std::any_of +#include // std::is_same, std::enable_if + +#include "tbb/blocked_range.h" + +namespace tbb { +namespace internal { + +/* + The blocked_rangeNd_impl uses make_index_sequence to automatically generate a ctor with + exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity + to use braced-init-list parameters to initialize each dimension. + Use of parameters, whose representation is a braced-init-list, but they're not + std::initializer_list or a reference to one, produces a non-deduced context + within template argument deduction. + + NOTE: blocked_rangeNd must be exactly a templated alias to the blocked_rangeNd_impl + (and not e.g. a derived class), otherwise it would need to declare its own ctor + facing the same problem that the impl class solves. +*/ + +template> +class blocked_rangeNd_impl; + +template +class blocked_rangeNd_impl> { +public: + //! Type of a value. + using value_type = Value; + +private: + + //! Helper type to construct range with N tbb::blocked_range objects. + template + using dim_type_helper = tbb::blocked_range; + +public: + blocked_rangeNd_impl() = delete; + + //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range. + blocked_rangeNd_impl(const dim_type_helper&... args) : my_dims{ {args...} } {} + + //! Dimensionality of a range. + static constexpr unsigned int ndims() { return N; } + + //! Range in certain dimension. + const tbb::blocked_range& dim(unsigned int dimension) const { + __TBB_ASSERT(dimension < N, "out of bound"); + return my_dims[dimension]; + } + + //------------------------------------------------------------------------ + // Methods that implement Range concept + //------------------------------------------------------------------------ + + //! True if at least one dimension is empty. + bool empty() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.empty(); + }); + } + + //! True if at least one dimension is divisible. + bool is_divisible() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.is_divisible(); + }); + } + +#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES + //! Static field to support proportional split. + static const bool is_splittable_in_proportion = true; + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, proportional_split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } +#endif + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + +private: + __TBB_STATIC_ASSERT(N != 0, "zero dimensional blocked_rangeNd can't be constructed"); + + //! Ranges in each dimension. + std::array, N> my_dims; + + template + void do_split(blocked_rangeNd_impl& r, split_type proportion) { + __TBB_STATIC_ASSERT((is_same_type::value + || is_same_type::value), + "type of split object is incorrect"); + __TBB_ASSERT(r.is_divisible(), "can't split not divisible range"); + + auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& first, const tbb::blocked_range& second) { + return (first.size() * second.grainsize() < second.size() * first.grainsize()); + }); + + auto r_it = r.my_dims.begin() + (my_it - my_dims.begin()); + + my_it->my_begin = tbb::blocked_range::do_split(*r_it, proportion); + + // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to + // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept + __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin), + "blocked_range has been split incorrectly"); + } +}; + +} // namespace internal + +template +using blocked_rangeNd = internal::blocked_rangeNd_impl; + +} // namespace tbb + +#endif /* __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ +#endif /* __TBB_blocked_rangeNd_H */ diff --git a/inst/include/tbb_local/tbb/cache_aligned_allocator.h b/inst/include/tbb_local/tbb/cache_aligned_allocator.h index d435e785..a9983298 100644 --- a/inst/include/tbb_local/tbb/cache_aligned_allocator.h +++ b/inst/include/tbb_local/tbb/cache_aligned_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_cache_aligned_allocator_H @@ -24,7 +20,11 @@ #include #include "tbb_stddef.h" #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward +#include // std::forward +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include #endif namespace tbb { @@ -69,7 +69,6 @@ class cache_aligned_allocator { template struct rebind { typedef cache_aligned_allocator other; }; - cache_aligned_allocator() throw() {} cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} template cache_aligned_allocator(const cache_aligned_allocator&) throw() {} @@ -132,6 +131,79 @@ inline bool operator==( const cache_aligned_allocator&, const cache_aligned_a template inline bool operator!=( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return false;} +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource wrapper to ensure cache line size alignment +class cache_aligned_resource : public std::pmr::memory_resource { +public: + cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} + explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} + + std::pmr::memory_resource* upstream_resource() const { + return m_upstream; + } + +private: + //! We don't know what memory resource set. Use padding to guarantee alignment + void* do_allocate(size_t bytes, size_t alignment) override { + size_t cache_line_alignment = correct_alignment(alignment); + uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment); + __TBB_ASSERT(base != 0, "Upstream resource returned NULL."); +#if _MSC_VER && !defined(__INTEL_COMPILER) + // unary minus operator applied to unsigned type, result still unsigned + #pragma warning(push) + #pragma warning(disable: 4146 4706) +#endif + // Round up to the next cache line (align the base address) + uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment; +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + // Record where block actually starts. + ((uintptr_t*)result)[-1] = base; + return (void*)result; + } + + void do_deallocate(void* ptr, size_t bytes, size_t alignment) override { + if (ptr) { + // Recover where block actually starts + uintptr_t base = ((uintptr_t*)ptr)[-1]; + m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment)); + } + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + if (this == &other) { return true; } +#if __TBB_USE_OPTIONAL_RTTI + const cache_aligned_resource* other_res = dynamic_cast(&other); + return other_res && (this->upstream_resource() == other_res->upstream_resource()); +#else + return false; +#endif + } + + size_t correct_alignment(size_t alignment) { + __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2"); +#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT + size_t cache_line_size = std::hardware_destructive_interference_size; +#else + size_t cache_line_size = internal::NFS_GetLineSize(); +#endif + return alignment < cache_line_size ? cache_line_size : alignment; + } + + size_t correct_size(size_t bytes) { + // To handle the case, when small size requested. There could be not + // enough space to store the original pointer. + return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes; + } + + std::pmr::memory_resource* m_upstream; +}; + +#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ + } // namespace tbb #endif /* __TBB_cache_aligned_allocator_H */ + diff --git a/inst/include/tbb_local/tbb/combinable.h b/inst/include/tbb_local/tbb/combinable.h index 0063dbb4..a8aaf61c 100644 --- a/inst/include/tbb_local/tbb/combinable.h +++ b/inst/include/tbb_local/tbb/combinable.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_combinable_H diff --git a/inst/include/tbb_local/tbb/compat/condition_variable b/inst/include/tbb_local/tbb/compat/condition_variable index 43edfc03..8dc4e913 100644 --- a/inst/include/tbb_local/tbb/compat/condition_variable +++ b/inst/include/tbb_local/tbb/compat/condition_variable @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_condition_variable_H diff --git a/inst/include/tbb_local/tbb/compat/ppl.h b/inst/include/tbb_local/tbb/compat/ppl.h index 840dfb22..a134244a 100644 --- a/inst/include/tbb_local/tbb/compat/ppl.h +++ b/inst/include/tbb_local/tbb/compat/ppl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_compat_ppl_H diff --git a/inst/include/tbb_local/tbb/compat/thread b/inst/include/tbb_local/tbb/compat/thread index 0edd9289..9dac4305 100644 --- a/inst/include/tbb_local/tbb/compat/thread +++ b/inst/include/tbb_local/tbb/compat/thread @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_thread_H diff --git a/inst/include/tbb_local/tbb/compat/tuple b/inst/include/tbb_local/tbb/compat/tuple index 5767c49e..86e6cd9f 100644 --- a/inst/include/tbb_local/tbb/compat/tuple +++ b/inst/include/tbb_local/tbb/compat/tuple @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tuple_H diff --git a/inst/include/tbb_local/tbb/concurrent_hash_map.h b/inst/include/tbb_local/tbb/concurrent_hash_map.h index 8497c838..e3f2a401 100644 --- a/inst/include/tbb_local/tbb/concurrent_hash_map.h +++ b/inst/include/tbb_local/tbb/concurrent_hash_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,39 +12,26 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_hash_map_H #define __TBB_concurrent_hash_map_H #include "tbb_stddef.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include #include // Need std::pair #include // Need std::memset #include __TBB_STD_SWAP_HEADER -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - -#include "cache_aligned_allocator.h" #include "tbb_allocator.h" #include "spin_rw_mutex.h" #include "atomic.h" #include "tbb_exception.h" #include "tbb_profiling.h" +#include "aligned_space.h" #include "internal/_tbb_hash_compare_impl.h" +#include "internal/_template_helpers.h" +#include "internal/_allocator_traits.h" #if __TBB_INITIALIZER_LISTS_PRESENT #include #endif @@ -54,12 +41,17 @@ #if __TBB_STATISTICS #include #endif +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT +// Definition of __TBB_CPP11_RVALUE_REF_PRESENT includes __TBB_CPP11_TUPLE_PRESENT +// for most of platforms, tuple present macro was added for logical correctness +#include +#endif namespace tbb { namespace interface5 { - template, typename A = tbb_allocator > > + template, typename A = tbb_allocator > > class concurrent_hash_map; //! @cond INTERNAL @@ -130,7 +122,7 @@ namespace interface5 { #endif //! Constructor hash_map_base() { - std::memset( (void*) this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 + std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 + sizeof(my_size) + sizeof(my_mask) // 4+4 or 8+8 + embedded_buckets*sizeof(bucket) ); // n*8 or n*16 for( size_type i = 0; i < embedded_block; i++ ) // fill the table @@ -166,7 +158,7 @@ namespace interface5 { //! Initialize buckets static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) { - if( is_initial ) std::memset((void*) ptr, 0, sz*sizeof(bucket) ); + if( is_initial ) std::memset( static_cast(ptr), 0, sz*sizeof(bucket) ); else for(size_type i = 0; i < sz; i++, ptr++) { *reinterpret_cast(&ptr->mutex) = 0; ptr->node_list = rehash_req; @@ -190,22 +182,25 @@ namespace interface5 { }; //! Enable segment - void enable_segment( segment_index_t k, bool is_initial = false ) { + template + void enable_segment( segment_index_t k, const Allocator& allocator, bool is_initial = false ) { + typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; + typedef tbb::internal::allocator_traits bucket_allocator_traits; + bucket_allocator_type bucket_allocator(allocator); __TBB_ASSERT( k, "Zero segment must be embedded" ); enable_segment_failsafe watchdog( my_table, k ); - cache_aligned_allocator alloc; size_type sz; __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment"); if( k >= first_block ) { sz = segment_size( k ); - segment_ptr_t ptr = alloc.allocate( sz ); + segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz); init_buckets( ptr, sz, is_initial ); itt_hide_store_word( my_table[k], ptr ); sz <<= 1;// double it to get entire capacity of the container } else { // the first block __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); sz = segment_size( first_block ); - segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets ); + segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz - embedded_buckets); init_buckets( ptr, sz - embedded_buckets, is_initial ); ptr -= segment_base(embedded_block); for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets @@ -215,6 +210,22 @@ namespace interface5 { watchdog.my_segment_ptr = 0; } + template + void delete_segment(segment_index_t s, const Allocator& allocator) { + typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; + typedef tbb::internal::allocator_traits bucket_allocator_traits; + bucket_allocator_type bucket_allocator(allocator); + segment_ptr_t buckets_ptr = my_table[s]; + size_type sz = segment_size( s ? s : 1 ); + + if( s >= first_block) // the first segment or the next + bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, sz); + else if( s == embedded_block && embedded_block != first_block ) + bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, + segment_size(first_block) - embedded_buckets); + if( s >= embedded_block ) my_table[s] = 0; + } + //! Get bucket by (masked) hashcode bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere? segment_index_t s = segment_index_of( h ); @@ -283,11 +294,12 @@ namespace interface5 { } //! Prepare enough segments for number of buckets - void reserve(size_type buckets) { + template + void reserve(size_type buckets, const Allocator& allocator) { if( !buckets-- ) return; bool is_initial = !my_size; for( size_type m = my_mask; buckets > m; m = my_mask ) - enable_segment( segment_index_of( m+1 ), is_initial ); + enable_segment( segment_index_of( m+1 ), allocator, is_initial ); } //! Swap hash_map_bases void internal_swap(hash_map_base &table) { @@ -299,6 +311,25 @@ namespace interface5 { for(size_type i = embedded_block; i < pointers_per_table; i++) swap(this->my_table[i], table.my_table[i]); } + +#if __TBB_CPP11_RVALUE_REF_PRESENT + void internal_move(hash_map_base&& other) { + my_mask = other.my_mask; + other.my_mask = embedded_buckets - 1; + my_size = other.my_size; + other.my_size = 0; + + for(size_type i = 0; i < embedded_buckets; ++i) { + my_embedded_segment[i].node_list = other.my_embedded_segment[i].node_list; + other.my_embedded_segment[i].node_list = NULL; + } + + for(size_type i = embedded_block; i < pointers_per_table; ++i) { + my_table[i] = other.my_table[i]; + other.my_table[i] = NULL; + } + } +#endif // __TBB_CPP11_RVALUE_REF_PRESENT }; template @@ -378,7 +409,7 @@ namespace interface5 { {} Value& operator*() const { __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); - return my_node->item; + return my_node->value(); } Value* operator->() const {return &operator*();} hash_map_iterator& operator++(); @@ -564,62 +595,78 @@ class concurrent_hash_map : protected internal::hash_map_base { protected: friend class const_accessor; - struct node; - typedef typename Allocator::template rebind::other node_allocator_type; + class node; + typedef typename tbb::internal::allocator_rebind::type node_allocator_type; + typedef tbb::internal::allocator_traits node_allocator_traits; node_allocator_type my_allocator; HashCompare my_hash_compare; - struct node : public node_base { - value_type item; - node( const Key &key ) : item(key, T()) {} - node( const Key &key, const T &t ) : item(key, t) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - node( const Key &key, T &&t ) : item(key, std::move(t)) {} - node( value_type&& i ) : item(std::move(i)){} -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - node( Args&&... args ) : item(std::forward(args)...) {} -#if __TBB_COPY_FROM_NON_CONST_REF_BROKEN - node( value_type& i ) : item(const_cast(i)) {} -#endif //__TBB_COPY_FROM_NON_CONST_REF_BROKEN -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - node( const value_type& i ) : item(i) {} - - // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17 - void *operator new( size_t /*size*/, node_allocator_type &a ) { - void *ptr = a.allocate(1); - if(!ptr) - tbb::internal::throw_exception(tbb::internal::eid_bad_alloc); - return ptr; - } - // match placement-new form above to be called if exception thrown in constructor - void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast(ptr),1); } + class node : public node_base { + tbb::aligned_space my_value; + public: + value_type* storage() { return my_value.begin(); } + value_type& value() { return *storage(); } }; void delete_node( node_base *n ) { - my_allocator.destroy( static_cast(n) ); - my_allocator.deallocate( static_cast(n), 1); + node_allocator_traits::destroy(my_allocator, static_cast(n)->storage()); + node_allocator_traits::destroy(my_allocator, static_cast(n)); + node_allocator_traits::deallocate(my_allocator, static_cast(n), 1); + } + + struct node_scoped_guard : tbb::internal::no_copy { + node* my_node; + node_allocator_type& my_alloc; + + node_scoped_guard(node* n, node_allocator_type& alloc) : my_node(n), my_alloc(alloc) {} + ~node_scoped_guard() { + if(my_node) { + node_allocator_traits::destroy(my_alloc, my_node); + node_allocator_traits::deallocate(my_alloc, my_node, 1); + } + } + void dismiss() { my_node = NULL; } + }; + +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + template + static node* create_node(node_allocator_type& allocator, Args&&... args) +#else + template + static node* create_node(node_allocator_type& allocator, __TBB_FORWARDING_REF(Arg1) arg1, __TBB_FORWARDING_REF(Arg2) arg2) +#endif + { + node* node_ptr = node_allocator_traits::allocate(allocator, 1); + node_scoped_guard guard(node_ptr, allocator); + node_allocator_traits::construct(allocator, node_ptr); +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + node_allocator_traits::construct(allocator, node_ptr->storage(), std::forward(args)...); +#else + node_allocator_traits::construct(allocator, node_ptr->storage(), tbb::internal::forward(arg1), tbb::internal::forward(arg2)); +#endif + guard.dismiss(); + return node_ptr; } static node* allocate_node_copy_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, *t); + return create_node(allocator, key, *t); } #if __TBB_CPP11_RVALUE_REF_PRESENT static node* allocate_node_move_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, std::move(*const_cast(t))); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static node* allocate_node_emplace_construct(node_allocator_type& allocator, Args&&... args){ - return new( allocator ) node(std::forward(args)...); + return create_node(allocator, key, std::move(*const_cast(t))); } -#endif //#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif static node* allocate_node_default_construct(node_allocator_type& allocator, const Key &key, const T * ){ - return new( allocator ) node(key); +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT + // Emplace construct an empty T object inside the pair + return create_node(allocator, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); +#else + T obj; // Use of temporary object in impossible, because create_node takes non-const reference + return create_node(allocator, key, tbb::internal::move(obj)); +#endif } static node* do_not_allocate_node(node_allocator_type& , const Key &, const T * ){ @@ -629,7 +676,7 @@ class concurrent_hash_map : protected internal::hash_map_base { node *search_bucket( const key_type &key, bucket *b ) const { node *n = static_cast( b->node_list ); - while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) ) + while( is_valid(n) && !my_hash_compare.equal(key, n->value().first) ) n = static_cast( n->next ); __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket"); return n; @@ -674,7 +721,7 @@ class concurrent_hash_map : protected internal::hash_map_base { __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL ); restart: for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(n)->item.first ); + hashcode_t c = my_hash_compare.hash( static_cast(n)->value().first ); #if TBB_USE_ASSERT hashcode_t bmask = h & (mask>>1); bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket @@ -726,7 +773,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Return reference to associated value in hash table. const_reference operator*() const { __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); - return my_node->item; + return my_node->value(); } //! Return pointer to associated value in hash table. @@ -756,7 +803,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Return reference to associated value in hash table. reference operator*() const { __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); - return this->my_node->item; + return this->my_node->value(); } //! Return pointer to associated value in hash table. @@ -770,18 +817,39 @@ class concurrent_hash_map : protected internal::hash_map_base { : internal::hash_map_base(), my_allocator(a) {} + explicit concurrent_hash_map( const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + {} + //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) + { + reserve( n, my_allocator ); + } + + concurrent_hash_map( size_type n, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) { - reserve( n ); + reserve( n, my_allocator ); } //! Copy constructor - concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a = allocator_type() ) + concurrent_hash_map( const concurrent_hash_map &table ) + : internal::hash_map_base(), + my_allocator(node_allocator_traits::select_on_container_copy_construction(table.get_allocator())) + { + call_clear_on_leave scope_guard(this); + internal_copy(table); + scope_guard.dismiss(); + } + + concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a) : internal::hash_map_base(), my_allocator(a) { + call_clear_on_leave scope_guard(this); internal_copy(table); + scope_guard.dismiss(); } #if __TBB_CPP11_RVALUE_REF_PRESENT @@ -789,7 +857,7 @@ class concurrent_hash_map : protected internal::hash_map_base { concurrent_hash_map( concurrent_hash_map &&table ) : internal::hash_map_base(), my_allocator(std::move(table.get_allocator())) { - swap(table); + internal_move(std::move(table)); } //! Move constructor @@ -797,10 +865,10 @@ class concurrent_hash_map : protected internal::hash_map_base { : internal::hash_map_base(), my_allocator(a) { if (a == table.get_allocator()){ - this->swap(table); + internal_move(std::move(table)); }else{ call_clear_on_leave scope_guard(this); - internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end())); + internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end()), table.size()); scope_guard.dismiss(); } } @@ -809,19 +877,38 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Construction with copying iteration range and given allocator instance template concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) + { + call_clear_on_leave scope_guard(this); + internal_copy(first, last, std::distance(first, last)); + scope_guard.dismiss(); + } + + template + concurrent_hash_map( I first, I last, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) { - reserve( std::distance(first, last) ); // TODO: load_factor? - internal_copy(first, last); + call_clear_on_leave scope_guard(this); + internal_copy(first, last, std::distance(first, last)); + scope_guard.dismiss(); } #if __TBB_INITIALIZER_LISTS_PRESENT //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. concurrent_hash_map( std::initializer_list il, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) { - reserve(il.size()); - internal_copy(il.begin(), il.end()); + call_clear_on_leave scope_guard(this); + internal_copy(il.begin(), il.end(), il.size()); + scope_guard.dismiss(); + } + + concurrent_hash_map( std::initializer_list il, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + { + call_clear_on_leave scope_guard(this); + internal_copy(il.begin(), il.end(), il.size()); + scope_guard.dismiss(); } #endif //__TBB_INITIALIZER_LISTS_PRESENT @@ -829,7 +916,9 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Assignment concurrent_hash_map& operator=( const concurrent_hash_map &table ) { if( this!=&table ) { + typedef typename node_allocator_traits::propagate_on_container_copy_assignment pocca_type; clear(); + tbb::internal::allocator_copy_assignment(my_allocator, table.my_allocator, pocca_type()); internal_copy(table); } return *this; @@ -838,17 +927,9 @@ class concurrent_hash_map : protected internal::hash_map_base { #if __TBB_CPP11_RVALUE_REF_PRESENT //! Move Assignment concurrent_hash_map& operator=( concurrent_hash_map &&table ) { - if(this != &table){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == table.my_allocator) { - concurrent_hash_map trash (std::move(*this)); - //TODO: swapping allocators here may be a problem, replace with single direction moving iff pocma is set - this->swap(table); - } else { - //do per element move - concurrent_hash_map moved_copy(std::move(table), this->my_allocator); - this->swap(moved_copy); - } + if(this != &table) { + typedef typename node_allocator_traits::propagate_on_container_move_assignment pocma_type; + internal_move_assign(std::move(table), pocma_type()); } return *this; } @@ -858,8 +939,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Assignment concurrent_hash_map& operator=( std::initializer_list il ) { clear(); - reserve(il.size()); - internal_copy(il.begin(), il.end()); + internal_copy(il.begin(), il.end(), il.size()); return *this; } #endif //__TBB_INITIALIZER_LISTS_PRESENT @@ -1067,8 +1147,8 @@ class concurrent_hash_map : protected internal::hash_map_base { template bool generic_emplace( Accessor && result, Args &&... args ) { result.release(); - node * node_ptr = allocate_node_emplace_construct(my_allocator, std::forward(args)...); - return lookup(/*insert*/true, node_ptr->item.first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); + node * node_ptr = create_node(my_allocator, std::forward(args)...); + return lookup(/*insert*/true, node_ptr->value().first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); } #endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif //__TBB_CPP11_RVALUE_REF_PRESENT @@ -1084,7 +1164,24 @@ class concurrent_hash_map : protected internal::hash_map_base { void internal_copy( const concurrent_hash_map& source ); template - void internal_copy( I first, I last ); + void internal_copy( I first, I last, size_type reserve_size ); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + // A compile-time dispatch to allow move assignment of containers with non-movable value_type if POCMA is true_type + void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_true_type) { + tbb::internal::allocator_move_assignment(my_allocator, other.my_allocator, tbb::internal::traits_true_type()); + internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_false_type) { + if (this->my_allocator == other.my_allocator) { + internal_move(std::move(other)); + } else { + //do per element move + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), other.size()); + } + } +#endif //! Fast find when no concurrent erasure is used. For internal use inside TBB only! /** Return pointer to item with given key, or NULL if no such item exists. @@ -1116,6 +1213,33 @@ class concurrent_hash_map : protected internal::hash_map_base { } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +namespace internal { +using namespace tbb::internal; + +template typename Map, typename Key, typename T, typename... Args> +using hash_map_t = Map< + Key, T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash_compare >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > > +>; +} + +// Deduction guide for the constructor from two iterators and hash_compare/ allocator +template +concurrent_hash_map(I, I, Args...) +-> internal::hash_map_t,internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list and hash_compare/ allocator +// Deduction guide for an initializer_list, hash_compare and allocator is implicit +template +concurrent_hash_map(std::initializer_list>, CompareOrAllocator) +-> internal::hash_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template bool concurrent_hash_map::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key&, const T*), node *tmp_n ) { __TBB_ASSERT( !result || !result->my_node, NULL ); @@ -1188,7 +1312,7 @@ bool concurrent_hash_map::lookup( bool op_insert, const Key #if __TBB_STATISTICS my_info_resizes++; // concurrent ones #endif - enable_segment( grow_segment ); + enable_segment( grow_segment, my_allocator ); } if( tmp_n ) // if op_insert only delete_node( tmp_n ); @@ -1256,7 +1380,7 @@ bool concurrent_hash_map::erase( const Key &key ) { search: node_base **p = &b()->node_list; n = *p; - while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->item.first ) ) { + while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->value().first ) ) { p = &n->next; n = *p; } @@ -1283,16 +1407,18 @@ bool concurrent_hash_map::erase( const Key &key ) { template void concurrent_hash_map::swap(concurrent_hash_map &table) { - //TODO: respect C++11 allocator_traits::propogate_on_constainer_swap - using std::swap; - swap(this->my_allocator, table.my_allocator); - swap(this->my_hash_compare, table.my_hash_compare); - internal_swap(table); + typedef typename node_allocator_traits::propagate_on_container_swap pocs_type; + if (this != &table && (pocs_type::value || my_allocator == table.my_allocator)) { + using std::swap; + tbb::internal::allocator_swap(this->my_allocator, table.my_allocator, pocs_type()); + swap(this->my_hash_compare, table.my_hash_compare); + internal_swap(table); + } } template void concurrent_hash_map::rehash(size_type sz) { - reserve( sz ); // TODO: add reduction of number of buckets as well + reserve( sz, my_allocator ); // TODO: add reduction of number of buckets as well hashcode_t mask = my_mask; hashcode_t b = (mask+1)>>1; // size or first index of the last segment __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2 @@ -1311,7 +1437,7 @@ void concurrent_hash_map::rehash(size_type sz) { // now h - is index of the root rehashed bucket b_old mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(q)->item.first ); + hashcode_t c = my_hash_compare.hash( static_cast(q)->value().first ); if( (c & mask) != h ) { // should be rehashed *p = q->next; // exclude from b_old bucket *b_new = get_bucket( c & mask ); @@ -1338,7 +1464,7 @@ void concurrent_hash_map::rehash(size_type sz) { #endif #if TBB_USE_ASSERT for( ; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ) & mask; + hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ) & mask; __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" ); } #endif @@ -1385,7 +1511,7 @@ void concurrent_hash_map::clear() { #endif #if __TBB_EXTRA_DEBUG for(; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ); + hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ); h &= m; __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" ); } @@ -1415,11 +1541,10 @@ void concurrent_hash_map::clear() { reported = true; } #endif -#endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS +#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS my_size = 0; segment_index_t s = segment_index_of( m ); __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" ); - cache_aligned_allocator alloc; do { __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" ); segment_ptr_t buckets_ptr = my_table[s]; @@ -1429,20 +1554,16 @@ void concurrent_hash_map::clear() { buckets_ptr[i].node_list = n->next; delete_node( n ); } - if( s >= first_block) // the first segment or the next - alloc.deallocate( buckets_ptr, sz ); - else if( s == embedded_block && embedded_block != first_block ) - alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets ); - if( s >= embedded_block ) my_table[s] = 0; + delete_segment(s, my_allocator); } while(s-- > 0); my_mask = embedded_buckets - 1; } template void concurrent_hash_map::internal_copy( const concurrent_hash_map& source ) { - reserve( source.my_size ); // TODO: load_factor? hashcode_t mask = source.my_mask; if( my_mask == mask ) { // optimized version + reserve( source.my_size, my_allocator ); // TODO: load_factor? bucket *dst = 0, *src = 0; bool rehash_required = false; for( hashcode_t k = 0; k <= mask; k++ ) { @@ -1454,24 +1575,26 @@ void concurrent_hash_map::internal_copy( const concurrent_h rehash_required = true; dst->node_list = internal::rehash_req; } else for(; n; n = static_cast( n->next ) ) { - add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) ); + node* node_ptr = create_node(my_allocator, n->value().first, n->value().second); + add_to_bucket( dst, node_ptr); ++my_size; // TODO: replace by non-atomic op } } if( rehash_required ) rehash(); - } else internal_copy( source.begin(), source.end() ); + } else internal_copy( source.begin(), source.end(), source.my_size ); } template template -void concurrent_hash_map::internal_copy(I first, I last) { +void concurrent_hash_map::internal_copy(I first, I last, size_type reserve_size) { + reserve( reserve_size, my_allocator ); // TODO: load_factor? hashcode_t m = my_mask; for(; first != last; ++first) { hashcode_t h = my_hash_compare.hash( (*first).first ); bucket *b = get_bucket( h & m ); __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = new( my_allocator ) node(*first); - add_to_bucket( b, n ); + node* node_ptr = create_node(my_allocator, (*first).first, (*first).second); + add_to_bucket( b, node_ptr ); ++my_size; // TODO: replace by non-atomic op } } diff --git a/inst/include/tbb_local/tbb/concurrent_lru_cache.h b/inst/include/tbb_local/tbb/concurrent_lru_cache.h index 8aacf241..c28fb6e2 100644 --- a/inst/include/tbb_local/tbb/concurrent_lru_cache.h +++ b/inst/include/tbb_local/tbb/concurrent_lru_cache.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_lru_cache_H @@ -25,10 +21,15 @@ #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h #endif +#include "tbb_stddef.h" + #include #include +#include // std::find +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include // std::move +#endif -#include "tbb_stddef.h" #include "atomic.h" #include "internal/_aggregator_impl.h" @@ -99,37 +100,84 @@ class concurrent_lru_cache : internal::no_assign{ } private: +#if !__TBB_CPP11_RVALUE_REF_PRESENT struct handle_move_t:no_assign{ concurrent_lru_cache & my_cache_ref; typename map_storage_type::reference my_map_record_ref; handle_move_t(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_ref(cache_ref),my_map_record_ref(value_ref) {}; }; +#endif class handle_object { concurrent_lru_cache * my_cache_pointer; - typename map_storage_type::reference my_map_record_ref; + typename map_storage_type::pointer my_map_record_ptr; public: - handle_object(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_pointer(&cache_ref), my_map_record_ref(value_ref) {} - handle_object(handle_move_t m):my_cache_pointer(&m.my_cache_ref), my_map_record_ref(m.my_map_record_ref){} - operator handle_move_t(){ return move(*this);} + handle_object() : my_cache_pointer(), my_map_record_ptr() {} + handle_object(concurrent_lru_cache& cache_ref, typename map_storage_type::reference value_ref) : my_cache_pointer(&cache_ref), my_map_record_ptr(&value_ref) {} + operator bool() const { + return (my_cache_pointer && my_map_record_ptr); + } +#if __TBB_CPP11_RVALUE_REF_PRESENT + // TODO: add check for double moved objects by special dedicated field + handle_object(handle_object&& src) : my_cache_pointer(src.my_cache_pointer), my_map_record_ptr(src.my_map_record_ptr) { + __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); + src.my_cache_pointer = NULL; + src.my_map_record_ptr = NULL; + } + handle_object& operator=(handle_object&& src) { + __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); + if (my_cache_pointer) { + my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); + } + my_cache_pointer = src.my_cache_pointer; + my_map_record_ptr = src.my_map_record_ptr; + src.my_cache_pointer = NULL; + src.my_map_record_ptr = NULL; + return *this; + } +#else + handle_object(handle_move_t m) : my_cache_pointer(&m.my_cache_ref), my_map_record_ptr(&m.my_map_record_ref) {} + handle_object& operator=(handle_move_t m) { + if (my_cache_pointer) { + my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); + } + my_cache_pointer = &m.my_cache_ref; + my_map_record_ptr = &m.my_map_record_ref; + return *this; + } + operator handle_move_t(){ + return move(*this); + } +#endif // __TBB_CPP11_RVALUE_REF_PRESENT value_type& value(){ - __TBB_ASSERT(my_cache_pointer,"get value from moved from object?"); - return my_map_record_ref.second.my_value; + __TBB_ASSERT(my_cache_pointer,"get value from already moved object?"); + __TBB_ASSERT(my_map_record_ptr,"get value from an invalid or already moved object?"); + return my_map_record_ptr->second.my_value; } ~handle_object(){ if (my_cache_pointer){ - my_cache_pointer->signal_end_of_usage(my_map_record_ref); + my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); } } private: +#if __TBB_CPP11_RVALUE_REF_PRESENT + // For source compatibility with C++03 + friend handle_object&& move(handle_object& h){ + return std::move(h); + } +#else friend handle_move_t move(handle_object& h){ return handle_object::move(h); } + // TODO: add check for double moved objects by special dedicated field static handle_move_t move(handle_object& h){ - __TBB_ASSERT(h.my_cache_pointer,"move from the same object twice ?"); + __TBB_ASSERT((h.my_cache_pointer && h.my_map_record_ptr) || (!h.my_cache_pointer && !h.my_map_record_ptr), "invalid state of moving object?"); concurrent_lru_cache * cache_pointer = h.my_cache_pointer; + typename map_storage_type::pointer map_record_ptr = h.my_map_record_ptr; h.my_cache_pointer = NULL; - return handle_move_t(*cache_pointer,h.my_map_record_ref); + h.my_map_record_ptr = NULL; + return handle_move_t(*cache_pointer, *map_record_ptr); } +#endif // __TBB_CPP11_RVALUE_REF_PRESENT private: void operator=(handle_object&); #if __SUNPRO_CC diff --git a/inst/include/tbb_local/tbb/concurrent_map.h b/inst/include/tbb_local/tbb/concurrent_map.h new file mode 100644 index 00000000..d022d880 --- /dev/null +++ b/inst/include/tbb_local/tbb/concurrent_map.h @@ -0,0 +1,383 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_map_H +#define __TBB_concurrent_map_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_map.h +#endif + +#include "tbb_config.h" + +// concurrent_map requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { + +namespace interface10 { + +template +class map_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using mapped_type = Value; + using compare_type = KeyCompare; + using value_type = std::pair; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + class value_compare { + public: + // TODO: these member types are deprecated in C++17, do we need to let them + using result_type = bool; + using first_argument_type = value_type; + using second_argument_type = value_type; + + bool operator()(const value_type& lhs, const value_type& rhs) const { + return comp(lhs.first, rhs.first); + } + + protected: + value_compare(compare_type c) : comp(c) {} + + friend class map_traits; + + compare_type comp; + }; + + static value_compare value_comp(compare_type comp) { return value_compare(comp); } + + static const key_type& get_key(const_reference val) { + return val.first; + } +}; // class map_traits + +template +class concurrent_multimap; + +template , typename Allocator = tbb_allocator>> +class concurrent_map + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = map_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_map() = default; + + explicit concurrent_map(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_map(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_map(const concurrent_map&) = default; + + concurrent_map(const concurrent_map& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_map(concurrent_map&&) = default; + + concurrent_map(concurrent_map&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_map(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_map(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_map& operator=(const concurrent_map& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_map& operator=(concurrent_map&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + mapped_type& at(const key_type& key) { + iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + const mapped_type& at(const key_type& key) const { + const_iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + mapped_type& operator[](const key_type& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + + return it->second; + } + + mapped_type& operator[](key_type&& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + + return it->second; + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Map, typename Key, typename T, typename... Args> +using c_map_t = Map 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > > >; +} // namespace internal + +template +concurrent_map(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_map(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator>> +class concurrent_multimap + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = map_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_multimap() = default; + + explicit concurrent_multimap(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multimap(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_multimap(const concurrent_multimap&) = default; + + concurrent_multimap(const concurrent_multimap& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multimap(concurrent_multimap&&) = default; + + concurrent_multimap(concurrent_multimap&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multimap(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multimap(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multimap& operator=(const concurrent_multimap& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multimap& operator=(concurrent_multimap&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + +}; // class concurrent_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +concurrent_multimap(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_multimap(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_map; +using interface10::concurrent_multimap; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_map_H diff --git a/inst/include/tbb_local/tbb/concurrent_priority_queue.h b/inst/include/tbb_local/tbb/concurrent_priority_queue.h index 89297d85..c5c03999 100644 --- a/inst/include/tbb_local/tbb/concurrent_priority_queue.h +++ b/inst/include/tbb_local/tbb/concurrent_priority_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_priority_queue_H @@ -27,6 +23,8 @@ #include "tbb_stddef.h" #include "tbb_profiling.h" #include "internal/_aggregator_impl.h" +#include "internal/_template_helpers.h" +#include "internal/_allocator_traits.h" #include #include #include @@ -85,14 +83,28 @@ class concurrent_priority_queue { typedef A allocator_type; //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), data(a) + explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(), data(a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + } + + //! Constructs a new concurrent_priority_queue with default capacity + explicit concurrent_priority_queue(const Compare& c, const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(c), data(a) { my_aggregator.initialize_handler(my_functor_t(this)); } //! Constructs a new concurrent_priority_queue with init_sz capacity explicit concurrent_priority_queue(size_type init_capacity, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), data(a) + mark(0), my_size(0), compare(), data(a) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(my_functor_t(this)); + } + + //! Constructs a new concurrent_priority_queue with init_sz capacity + explicit concurrent_priority_queue(size_type init_capacity, const Compare& c, const allocator_type& a = allocator_type()) : + mark(0), my_size(0), compare(c), data(a) { data.reserve(init_capacity); my_aggregator.initialize_handler(my_functor_t(this)); @@ -101,7 +113,17 @@ class concurrent_priority_queue { //! [begin,end) constructor template concurrent_priority_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - mark(0), data(begin, end, a) + mark(0), compare(), data(begin, end, a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + heapify(); + my_size = data.size(); + } + + //! [begin,end) constructor + template + concurrent_priority_queue(InputIterator begin, InputIterator end, const Compare& c, const allocator_type& a = allocator_type()) : + mark(0), compare(c), data(begin, end, a) { my_aggregator.initialize_handler(my_functor_t(this)); heapify(); @@ -111,7 +133,16 @@ class concurrent_priority_queue { #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from std::initializer_list concurrent_priority_queue(std::initializer_list init_list, const allocator_type &a = allocator_type()) : - mark(0),data(init_list.begin(), init_list.end(), a) + mark(0), compare(), data(init_list.begin(), init_list.end(), a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + heapify(); + my_size = data.size(); + } + + //! Constructor from std::initializer_list + concurrent_priority_queue(std::initializer_list init_list, const Compare& c, const allocator_type &a = allocator_type()) : + mark(0), compare(c), data(init_list.begin(), init_list.end(), a) { my_aggregator.initialize_handler(my_functor_t(this)); heapify(); @@ -121,7 +152,7 @@ class concurrent_priority_queue { //! Copy constructor /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - explicit concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), + concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), my_size(src.my_size), data(src.data.begin(), src.data.end(), src.data.get_allocator()) { my_aggregator.initialize_handler(my_functor_t(this)); @@ -481,6 +512,31 @@ class concurrent_priority_queue { } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +namespace internal { + +template +using priority_queue_t = concurrent_priority_queue< + T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, cache_aligned_allocator > +>; +} + +// Deduction guide for the constructor from two iterators +template::value_type, + typename... Args +> concurrent_priority_queue(InputIterator, InputIterator, Args...) +-> internal::priority_queue_t; + +template +concurrent_priority_queue(std::initializer_list init_list, CompareOrAllocalor) +-> internal::priority_queue_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_priority_queue; diff --git a/inst/include/tbb_local/tbb/concurrent_queue.h b/inst/include/tbb_local/tbb/concurrent_queue.h index 81db58a3..be8e8f81 100644 --- a/inst/include/tbb_local/tbb/concurrent_queue.h +++ b/inst/include/tbb_local/tbb/concurrent_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_queue_H #define __TBB_concurrent_queue_H #include "internal/_concurrent_queue_impl.h" +#include "internal/_allocator_traits.h" namespace tbb { @@ -36,7 +33,7 @@ class concurrent_queue: public internal::concurrent_queue_base_v3 { template friend class internal::concurrent_queue_iterator; //! Allocator type - typedef typename A::template rebind::other page_allocator_type; + typedef typename tbb::internal::allocator_rebind::type page_allocator_type; page_allocator_type my_allocator; //! Allocates a block of size n (bytes) @@ -177,6 +174,15 @@ class concurrent_queue: public internal::concurrent_queue_base_v3 { const_iterator unsafe_end() const {return const_iterator();} } ; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template::value_type, + typename A = cache_aligned_allocator +> concurrent_queue(InputIterator, InputIterator, const A& = A()) +-> concurrent_queue; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template concurrent_queue::~concurrent_queue() { clear(); @@ -200,9 +206,9 @@ void concurrent_queue::clear() { template > class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { template friend class internal::concurrent_queue_iterator; + typedef typename tbb::internal::allocator_rebind::type page_allocator_type; //! Allocator type - typedef typename A::template rebind::other page_allocator_type; page_allocator_type my_allocator; typedef typename concurrent_queue_base_v3::padded_page padded_page; @@ -439,6 +445,15 @@ class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// guide for concurrent_bounded_queue(InputIterator, InputIterator, ...) +template::value_type, + typename A = cache_aligned_allocator +> concurrent_bounded_queue(InputIterator, InputIterator, const A& = A()) +-> concurrent_bounded_queue; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template concurrent_bounded_queue::~concurrent_bounded_queue() { clear(); diff --git a/inst/include/tbb_local/tbb/concurrent_set.h b/inst/include/tbb_local/tbb/concurrent_set.h new file mode 100644 index 00000000..70269947 --- /dev/null +++ b/inst/include/tbb_local/tbb/concurrent_set.h @@ -0,0 +1,297 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_set_H +#define __TBB_concurrent_set_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_set.h +#endif + +#include "tbb/tbb_config.h" + +// concurrent_set requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { +namespace interface10 { + +// TODO: test this class +template +class set_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using value_type = key_type; + using compare_type = KeyCompare; + using value_compare = compare_type; + using reference = value_type & ; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + static const key_type& get_key(const_reference val) { + return val; + } + + static value_compare value_comp(compare_type comp) { return comp; } +}; + +template +class concurrent_multiset; + +template , typename Allocator = tbb_allocator> +class concurrent_set + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = set_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_set() = default; + + explicit concurrent_set(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_set(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_set(const concurrent_set&) = default; + + concurrent_set(const concurrent_set& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_set(concurrent_set&&) = default; + + concurrent_set(concurrent_set&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_set(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_set(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_set& operator=(const concurrent_set& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_set& operator=(concurrent_set&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Set, typename Key, typename... Args> +using c_set_t = Set 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > >; +} // namespace internal + +template +concurrent_set(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_set(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator> +class concurrent_multiset + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = set_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_multiset() = default; + + explicit concurrent_multiset(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multiset(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(first, last); + } + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const allocator_type& alloc) : base_type(key_compare(), alloc) { + insert(first, last); + } + + /** Copy constructor */ + concurrent_multiset(const concurrent_multiset&) = default; + + concurrent_multiset(const concurrent_multiset& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multiset(concurrent_multiset&&) = default; + + concurrent_multiset(concurrent_multiset&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multiset(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multiset(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multiset& operator=(const concurrent_multiset& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multiset& operator=(concurrent_multiset&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + + +template +concurrent_multiset(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_multiset(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_set; +using interface10::concurrent_multiset; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_set_H diff --git a/inst/include/tbb_local/tbb/concurrent_unordered_map.h b/inst/include/tbb_local/tbb/concurrent_unordered_map.h index c959a7ec..cc73dad7 100644 --- a/inst/include/tbb_local/tbb/concurrent_unordered_map.h +++ b/inst/include/tbb_local/tbb/concurrent_unordered_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -39,7 +35,13 @@ class concurrent_unordered_map_traits typedef std::pair value_type; typedef Key key_type; typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef tbb::internal::node_handle::node, + allocator_type> node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + enum { allow_multimapping = Allow_multimapping }; concurrent_unordered_map_traits() : my_hash_compare() {} @@ -53,6 +55,9 @@ class concurrent_unordered_map_traits hash_compare my_hash_compare; // the comparator predicate for keys }; +template +class concurrent_unordered_multimap; + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > > class concurrent_unordered_map : @@ -93,12 +98,23 @@ class concurrent_unordered_map : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_map(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_map(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_map(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_map(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -106,9 +122,24 @@ class concurrent_unordered_map : template concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } @@ -116,16 +147,30 @@ class concurrent_unordered_map : #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(il.begin(),il.end()); + } + + concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(), il.end()); } + + concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_map(const concurrent_unordered_map& table) : base_type(table) {} @@ -143,11 +188,31 @@ class concurrent_unordered_map : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_map(concurrent_unordered_map&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_map& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_map&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_map(const concurrent_unordered_map& table, const Allocator& a) : base_type(table, a) @@ -191,6 +256,45 @@ class concurrent_unordered_map : } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { +using namespace tbb::internal; + +template typename Map, typename Key, typename Element, typename... Args> +using cu_map_t = Map< + Key, Element, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash >, + std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, + pack_element_t<1, Args...>, std::equal_to >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > > +>; +} + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_map (I, I) +-> internal::cu_map_t, internal::iterator_mapped_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_map(I, I, size_t, Args...) +-> internal::cu_map_t, internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_map(std::initializer_list>) +-> internal::cu_map_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_map(std::initializer_list>, size_t, Args...) +-> internal::cu_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template < typename Key, typename T, typename Hasher = tbb::tbb_hash, typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > > class concurrent_unordered_multimap : @@ -229,12 +333,23 @@ class concurrent_unordered_multimap : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_multimap(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_multimap(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_multimap(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_multimap(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -242,9 +357,24 @@ class concurrent_unordered_multimap : template concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets,key_compare(_Hasher,_Key_equality), a) + : base_type(n_of_buckets,key_compare(a_hasher,a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } @@ -252,16 +382,29 @@ class concurrent_unordered_multimap : #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(il.begin(),il.end()); + } + + concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(), il.end()); } + + concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_multimap(const concurrent_unordered_multimap& table) : base_type(table) {} @@ -279,16 +422,60 @@ class concurrent_unordered_multimap : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_multimap(concurrent_unordered_multimap&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_map& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_map&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_multimap(const concurrent_unordered_multimap& table, const Allocator& a) : base_type(table, a) {} }; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_multimap (I, I) +-> internal::cu_map_t, internal::iterator_mapped_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_multimap(I, I, size_t, Args...) +-> internal::cu_map_t, internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_multimap(std::initializer_list>) +-> internal::cu_map_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_multimap(std::initializer_list>, size_t, Args...) +-> internal::cu_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_unordered_map; diff --git a/inst/include/tbb_local/tbb/concurrent_unordered_set.h b/inst/include/tbb_local/tbb/concurrent_unordered_set.h index a26fee1a..93a772ba 100644 --- a/inst/include/tbb_local/tbb/concurrent_unordered_set.h +++ b/inst/include/tbb_local/tbb/concurrent_unordered_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -39,7 +35,13 @@ class concurrent_unordered_set_traits typedef Key value_type; typedef Key key_type; typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef tbb::internal::node_handle::node, + allocator_type> node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + enum { allow_multimapping = Allow_multimapping }; concurrent_unordered_set_traits() : my_hash_compare() {} @@ -52,6 +54,9 @@ class concurrent_unordered_set_traits hash_compare my_hash_compare; // the comparator predicate for keys }; +template +class concurrent_unordered_multiset; + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > class concurrent_unordered_set : public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, false> > { @@ -87,6 +92,9 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif /*__TBB_UNORDERED_NODE_HANDLE_PRESENT*/ // Construction/destruction/copying explicit concurrent_unordered_set(size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), @@ -94,29 +102,63 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) {} + concurrent_unordered_set(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_set(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + {} + explicit concurrent_unordered_set(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) {} template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { insert(first, last); } + template + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(first, last); + } + #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(),il.end()); + } + + concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(il.begin(), il.end()); } + + concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_set(const concurrent_unordered_set& table) : base_type(table) {} @@ -134,12 +176,32 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_set(concurrent_unordered_set&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_set& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_set&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_set(const concurrent_unordered_set& table, const Allocator& a) : base_type(table, a) @@ -147,6 +209,45 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { +using namespace tbb::internal; + +template typename Set, typename T, typename... Args> +using cu_set_t = Set < + T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash >, + std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, + pack_element_t<1, Args...>, std::equal_to >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > +>; +} + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_set(I, I) +-> internal::cu_set_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_set(I, I, size_t, Args...) +-> internal::cu_set_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_set(std::initializer_list) +-> internal::cu_set_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_set(std::initializer_list, size_t, Args...) +-> internal::cu_set_t; + +#endif /*__TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > class concurrent_unordered_multiset : @@ -185,12 +286,24 @@ class concurrent_unordered_multiset : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_multiset(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_multiset(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_multiset(size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_multiset(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -198,25 +311,54 @@ class concurrent_unordered_multiset : template concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(),il.end()); + } + + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(il.begin(), il.end()); } + + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_multiset(const concurrent_unordered_multiset& table) : base_type(table) {} @@ -234,18 +376,62 @@ class concurrent_unordered_multiset : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_multiset(concurrent_unordered_multiset&& table, const Allocator& a) : base_type(std::move(table), a) { } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_set& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_set&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_multiset(const concurrent_unordered_multiset& table, const Allocator& a) : base_type(table, a) {} }; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_multiset(I, I) +-> internal::cu_set_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_multiset(I, I, size_t, Args...) +-> internal::cu_set_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_multiset(std::initializer_list) +-> internal::cu_set_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_multiset(std::initializer_list, size_t, Args...) +-> internal::cu_set_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_unordered_set; diff --git a/inst/include/tbb_local/tbb/concurrent_vector.h b/inst/include/tbb_local/tbb/concurrent_vector.h index bd93615e..461d4c35 100644 --- a/inst/include/tbb_local/tbb/concurrent_vector.h +++ b/inst/include/tbb_local/tbb/concurrent_vector.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_vector_H @@ -31,19 +27,10 @@ #include #include // for memset() #include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include #include -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif +#include "internal/_allocator_traits.h" #if _MSC_VER==1500 && !__INTEL_COMPILER // VS2008/VC9 seems to have an issue; limits pull in math.h @@ -85,7 +72,7 @@ namespace internal { //! Exception helper function template void handle_unconstructed_elements(T* array, size_t n_of_elements){ - std::memset( (void*) array, 0, n_of_elements * sizeof( T ) ); + std::memset( static_cast(array), 0, n_of_elements * sizeof( T ) ); } //! Base class of concurrent vector implementation. @@ -105,11 +92,11 @@ namespace internal { pointers_per_short_table = 3, // to fit into 8 words of entire structure pointers_per_long_table = sizeof(segment_index_t) * 8 // one segment per bit }; - public: + struct segment_not_used {}; struct segment_allocated {}; struct segment_allocation_failed {}; - protected: + class segment_t; class segment_value_t { void* array; @@ -478,12 +465,9 @@ namespace internal { template class allocator_base { public: - typedef typename A::template - rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; allocator_type my_allocator; - allocator_base(const allocator_type &a = allocator_type() ) : my_allocator(a) {} - }; } // namespace internal @@ -753,9 +737,7 @@ class concurrent_vector: protected internal::allocator_base, if(pocma_t::value || this->my_allocator == other.my_allocator) { concurrent_vector trash (std::move(*this)); internal_swap(other); - if (pocma_t::value) { - this->my_allocator = std::move(other.my_allocator); - } + tbb::internal::allocator_move_assignment(this->my_allocator, other.my_allocator, pocma_t()); } else { internal_assign(other, sizeof(T), &destroy_array, &move_assign_array, &move_array); } @@ -1013,10 +995,10 @@ class concurrent_vector: protected internal::allocator_base, //! swap two instances void swap(concurrent_vector &vector) { - using std::swap; - if( this != &vector ) { + typedef typename tbb::internal::allocator_traits::propagate_on_container_swap pocs_t; + if( this != &vector && (this->my_allocator == vector.my_allocator || pocs_t::value) ) { concurrent_vector_base_v3::internal_swap(static_cast(vector)); - swap(this->my_allocator, vector.my_allocator); + tbb::internal::allocator_swap(this->my_allocator, vector.my_allocator, pocs_t()); } } @@ -1053,7 +1035,13 @@ class concurrent_vector: protected internal::allocator_base, internal_resize( n, sizeof(T), max_size(), static_cast(p), &destroy_array, p? &initialize_array_by : &initialize_array ); } - //! helper class + //! True/false function override helper + /* Functions declarations: + * void foo(is_integer_tag*); + * void foo(is_integer_tag*); + * Usage example: + * foo(static_cast::is_integer>*>(0)); + */ template class is_integer_tag; //! assign integer items by copying when arguments are treated as iterators. See C++ Standard 2003 23.1.1p9 @@ -1167,6 +1155,25 @@ class concurrent_vector: protected internal::allocator_base, }; }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template::value_type, + typename A = cache_aligned_allocator +> concurrent_vector(I, I, const A& = A()) +-> concurrent_vector; + +// Deduction guide for the constructor from a vector and allocator +template +concurrent_vector(const concurrent_vector &, const A2 &) +-> concurrent_vector; + +// Deduction guide for the constructor from an initializer_list +template +> concurrent_vector(std::initializer_list, const A& = A()) +-> concurrent_vector; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) #pragma warning (push) #pragma warning (disable: 4701) // potentially uninitialized local variable "old" @@ -1304,8 +1311,8 @@ void concurrent_vector::move_array_if_noexcept( void* dst, const void* src template template void concurrent_vector::copy_range( void* dst, const void* p_type_erased_iterator, size_type n ){ - I & iterator ((*const_cast(static_cast(p_type_erased_iterator)))); - internal_loop_guide loop(n, dst); loop.iterate(iterator); + internal_loop_guide loop(n, dst); + loop.iterate( *(static_cast(const_cast(p_type_erased_iterator))) ); } template diff --git a/inst/include/tbb_local/tbb/critical_section.h b/inst/include/tbb_local/tbb/critical_section.h index 324b3e13..ba693b38 100644 --- a/inst/include/tbb_local/tbb/critical_section.h +++ b/inst/include/tbb_local/tbb/critical_section.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_CRITICAL_SECTION_H_ diff --git a/inst/include/tbb_local/tbb/enumerable_thread_specific.h b/inst/include/tbb_local/tbb/enumerable_thread_specific.h index f31f533b..1bcac28d 100644 --- a/inst/include/tbb_local/tbb/enumerable_thread_specific.h +++ b/inst/include/tbb_local/tbb/enumerable_thread_specific.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_enumerable_thread_specific_H diff --git a/inst/include/tbb_local/tbb/flow_graph.h b/inst/include/tbb_local/tbb/flow_graph.h index 7b03c0e2..cbdad306 100644 --- a/inst/include/tbb_local/tbb/flow_graph.h +++ b/inst/include/tbb_local/tbb/flow_graph.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_H @@ -34,7 +30,6 @@ #include "internal/_aggregator_impl.h" #include "tbb_profiling.h" #include "task_arena.h" -#include "flow_graph_abstractions.h" #if __TBB_PREVIEW_ASYNC_MSG #include // std::vector in internal::async_storage @@ -88,66 +83,56 @@ namespace flow { //! An enumeration the provides the two most common concurrency levels: unlimited and serial enum concurrency { unlimited = 0, serial = 1 }; -namespace internal { -static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; -} - -namespace interface9 { - -using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; +namespace interface10 { -namespace internal { - template class successor_cache; - template class broadcast_cache; - template class round_robin_cache; - template class predecessor_cache; - template class reservable_predecessor_cache; -} - -//A generic null type +//! A generic null type struct null_type {}; //! An empty class used for messages that mean "I'm done" class continue_msg {}; +//! Forward declaration section template< typename T > class sender; template< typename T > class receiver; class continue_receiver; - -template< typename T > class limiter_node; // needed for resetting decrementer +} // namespaceX +namespace interface11 { +template< typename T, typename U > class limiter_node; // needed for resetting decrementer +} +namespace interface10 { template< typename R, typename B > class run_and_put_task; -// flags to modify the behavior of the graph reset(). Can be combined. -enum reset_flags { - rf_reset_protocol = 0, - rf_reset_bodies = 1<<0, // delete the current node body, reset to a copy of the initial node body. - rf_clear_edges = 1<<1 // delete edges -}; +namespace internal { + +template class successor_cache; +template class broadcast_cache; +template class round_robin_cache; +template class predecessor_cache; +template class reservable_predecessor_cache; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES -//* holder of edges both for caches and for those nodes which do not have predecessor caches. +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION +// Holder of edges both for caches and for those nodes which do not have predecessor caches. // C == receiver< ... > or sender< ... >, depending. -namespace internal { template class edge_container { public: typedef std::list > edge_list_type; - void add_edge( C &s) { - built_edges.push_back( &s ); + void add_edge(C &s) { + built_edges.push_back(&s); } - void delete_edge( C &s) { - for ( typename edge_list_type::iterator i = built_edges.begin(); i != built_edges.end(); ++i ) { - if ( *i == &s ) { + void delete_edge(C &s) { + for (typename edge_list_type::iterator i = built_edges.begin(); i != built_edges.end(); ++i) { + if (*i == &s) { (void)built_edges.erase(i); return; // only remove one predecessor per request } } } - void copy_edges( edge_list_type &v) { + void copy_edges(edge_list_type &v) { v = built_edges; } @@ -161,21 +146,133 @@ class edge_container { // methods remove the statement from all predecessors/successors liste in the edge // container. - template< typename S > void sender_extract( S &s ); - template< typename R > void receiver_extract( R &r ); + template< typename S > void sender_extract(S &s); + template< typename R > void receiver_extract(R &r); -private: +private: edge_list_type built_edges; }; // class edge_container -} // namespace internal -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + +} // namespace internal + +} // namespace interface10 +} // namespace flow +} // namespace tbb + +//! The graph class +#include "internal/_flow_graph_impl.h" + +namespace tbb { +namespace flow { +namespace interface10 { + +// enqueue left task if necessary. Returns the non-enqueued task if there is one. +static inline tbb::task *combine_tasks(graph& g, tbb::task * left, tbb::task * right) { + // if no RHS task, don't change left. + if (right == NULL) return left; + // right != NULL + if (left == NULL) return right; + if (left == SUCCESSFULLY_ENQUEUED) return right; + // left contains a task + if (right != SUCCESSFULLY_ENQUEUED) { + // both are valid tasks + internal::spawn_in_graph_arena(g, *left); + return right; + } + return left; +} #if __TBB_PREVIEW_ASYNC_MSG -#include "internal/_flow_graph_async_msg_impl.h" +template < typename T > class async_msg; namespace internal { +template < typename T > class async_storage; + +template< typename T, typename = void > +struct async_helpers { + typedef async_msg async_type; + typedef T filtered_type; + + static const bool is_async_type = false; + + static const void* to_void_ptr(const T& t) { + return static_cast(&t); + } + + static void* to_void_ptr(T& t) { + return static_cast(&t); + } + + static const T& from_void_ptr(const void* p) { + return *static_cast(p); + } + + static T& from_void_ptr(void* p) { + return *static_cast(p); + } + + static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { + if (is_async) { + // This (T) is NOT async and incoming 'A t' IS async + // Get data from async_msg + const async_msg& msg = async_helpers< async_msg >::from_void_ptr(p); + task* const new_task = msg.my_storage->subscribe(*this_recv, this_recv->graph_reference()); + // finalize() must be called after subscribe() because set() can be called in finalize() + // and 'this_recv' client must be subscribed by this moment + msg.finalize(); + return new_task; + } + else { + // Incoming 't' is NOT async + return this_recv->try_put_task(from_void_ptr(p)); + } + } +}; + +template< typename T > +struct async_helpers< T, typename std::enable_if< std::is_base_of, T>::value >::type > { + typedef T async_type; + typedef typename T::async_msg_data_type filtered_type; + + static const bool is_async_type = true; + + // Receiver-classes use const interfaces + static const void* to_void_ptr(const T& t) { + return static_cast(&static_cast&>(t)); + } + + static void* to_void_ptr(T& t) { + return static_cast(&static_cast&>(t)); + } + + // Sender-classes use non-const interfaces + static const T& from_void_ptr(const void* p) { + return *static_cast(static_cast*>(p)); + } + + static T& from_void_ptr(void* p) { + return *static_cast(static_cast*>(p)); + } + + // Used in receiver class + static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { + if (is_async) { + // Both are async + return this_recv->try_put_task(from_void_ptr(p)); + } + else { + // This (T) is async and incoming 'X t' is NOT async + // Create async_msg for X + const filtered_type& t = async_helpers::from_void_ptr(p); + const T msg(t); + return this_recv->try_put_task(msg); + } + } +}; + class untyped_receiver; class untyped_sender { @@ -203,7 +300,7 @@ class untyped_sender { //! Consumes the reserved item virtual bool try_consume( ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef internal::edge_container built_successors_type; typedef built_successors_type::edge_list_type successor_list_type; @@ -212,7 +309,7 @@ class untyped_sender { virtual void internal_delete_built_successor( successor_type & ) = 0; virtual void copy_successors( successor_list_type &) = 0; virtual size_t successor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: //! Request an item from the sender template< typename X > @@ -232,7 +329,6 @@ class untyped_sender { class untyped_receiver { template< typename, typename > friend class run_and_put_task; - template< typename > friend class limiter_node; template< typename, typename > friend class internal::broadcast_cache; template< typename, typename > friend class internal::round_robin_cache; @@ -253,7 +349,7 @@ class untyped_receiver { bool try_put(const X& t) { task *res = try_put_task(t); if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); + if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); return true; } @@ -267,7 +363,7 @@ class untyped_receiver { //! Remove a predecessor from the node virtual bool remove_predecessor( predecessor_type & ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef internal::edge_container built_predecessors_type; typedef built_predecessors_type::edge_list_type predecessor_list_type; virtual built_predecessors_type &built_predecessors() = 0; @@ -275,7 +371,7 @@ class untyped_receiver { virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; virtual void copy_predecessors( predecessor_list_type & ) = 0; virtual size_t predecessor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template task *try_put_task(const X& t) { @@ -284,6 +380,8 @@ class untyped_receiver { virtual task* try_put_task_wrapper( const void* p, bool is_async ) = 0; + virtual graph& graph_reference() = 0; + // NOTE: Following part of PROTECTED and PRIVATE sections is copy-paste from original receiver class //! put receiver back in initial state @@ -395,7 +493,7 @@ class sender { //! Consumes the reserved item virtual bool try_consume( ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef typename internal::edge_container built_successors_type; typedef typename built_successors_type::edge_list_type successor_list_type; @@ -404,7 +502,7 @@ class sender { virtual void internal_delete_built_successor( successor_type & ) = 0; virtual void copy_successors( successor_list_type &) = 0; virtual size_t successor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ }; // class sender //! Pure virtual template class that defines a receiver of messages of type T @@ -424,7 +522,7 @@ class receiver { bool try_put( const T& t ) { task *res = try_put_task(t); if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); + if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); return true; } @@ -434,6 +532,7 @@ class receiver { template< typename X, typename Y > friend class internal::broadcast_cache; template< typename X, typename Y > friend class internal::round_robin_cache; virtual task *try_put_task(const T& t) = 0; + virtual graph& graph_reference() = 0; public: // NOTE: Following part of PUBLIC and PROTECTED sections is copy-pasted in receiver under #if __TBB_PREVIEW_ASYNC_MSG @@ -443,7 +542,7 @@ class receiver { //! Remove a predecessor from the node virtual bool remove_predecessor( predecessor_type & ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename internal::edge_container built_predecessors_type; typedef typename built_predecessors_type::edge_list_type predecessor_list_type; virtual built_predecessors_type &built_predecessors() = 0; @@ -451,11 +550,10 @@ class receiver { virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; virtual void copy_predecessors( predecessor_list_type & ) = 0; virtual size_t predecessor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: //! put receiver back in initial state - template friend class limiter_node; virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; template friend class internal::successor_cache; @@ -468,22 +566,6 @@ class receiver { #endif // __TBB_PREVIEW_ASYNC_MSG -// enqueue left task if necessary. Returns the non-enqueued task if there is one. -static inline tbb::task *combine_tasks( tbb::task * left, tbb::task * right) { - // if no RHS task, don't change left. - if(right == NULL) return left; - // right != NULL - if(left == NULL) return right; - if(left == SUCCESSFULLY_ENQUEUED) return right; - // left contains a task - if(right != SUCCESSFULLY_ENQUEUED) { - // both are valid tasks - FLOW_SPAWN(*left); - return right; - } - return left; -} - //! Base class for receivers of completion messages /** These receivers automatically reset, but cannot be explicitly waited on */ class continue_receiver : public receiver< continue_msg > { @@ -496,15 +578,18 @@ class continue_receiver : public receiver< continue_msg > { typedef receiver::predecessor_type predecessor_type; //! Constructor - explicit continue_receiver( int number_of_predecessors = 0 ) { + explicit continue_receiver( + __TBB_FLOW_GRAPH_PRIORITY_ARG1(int number_of_predecessors, node_priority_t priority)) { my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; my_current_count = 0; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = priority; ) } //! Copy constructor continue_receiver( const continue_receiver& src ) : receiver() { my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; my_current_count = 0; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = src.my_priority; ) } //! Increments the trigger threshold @@ -524,7 +609,7 @@ class continue_receiver : public receiver< continue_msg > { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef internal::edge_container built_predecessors_type; typedef built_predecessors_type::edge_list_type predecessor_list_type; built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } @@ -549,7 +634,7 @@ class continue_receiver : public receiver< continue_msg > { return my_built_predecessors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template< typename R, typename B > friend class run_and_put_task; @@ -568,7 +653,7 @@ class continue_receiver : public receiver< continue_msg > { return res? res : SUCCESSFULLY_ENQUEUED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // continue_receiver must contain its own built_predecessors because it does // not have a node_cache. built_predecessors_type my_built_predecessors; @@ -577,14 +662,15 @@ class continue_receiver : public receiver< continue_msg > { int my_predecessor_count; int my_current_count; int my_initial_predecessor_count; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) // the friend declaration in the base class did not eliminate the "protected class" // error in gcc 4.1.2 - template friend class limiter_node; + template friend class tbb::flow::interface11::limiter_node; void reset_receiver( reset_flags f ) __TBB_override { my_current_count = 0; if (f & rf_clear_edges) { -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_predecessors.clear(); #endif my_predecessor_count = my_initial_predecessor_count; @@ -599,7 +685,8 @@ class continue_receiver : public receiver< continue_msg > { bool is_continue_receiver() __TBB_override { return true; } }; // class continue_receiver -} // interface9 + +} // interfaceX #if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING template @@ -608,9 +695,9 @@ class continue_receiver : public receiver< continue_msg > { } #endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - using interface9::sender; - using interface9::receiver; - using interface9::continue_receiver; + using interface10::sender; + using interface10::receiver; + using interface10::continue_receiver; } // flow } // tbb @@ -619,317 +706,15 @@ class continue_receiver : public receiver< continue_msg > { namespace tbb { namespace flow { -namespace interface9 { +namespace interface10 { -#include "internal/_flow_graph_impl.h" +#include "internal/_flow_graph_body_impl.h" +#include "internal/_flow_graph_cache_impl.h" #include "internal/_flow_graph_types_impl.h" -using namespace internal::graph_policy_namespace; - -class graph; -class graph_node; - -template -class graph_iterator { - friend class graph; - friend class graph_node; -public: - typedef size_t size_type; - typedef GraphNodeType value_type; - typedef GraphNodeType* pointer; - typedef GraphNodeType& reference; - typedef const GraphNodeType& const_reference; - typedef std::forward_iterator_tag iterator_category; - - //! Default constructor - graph_iterator() : my_graph(NULL), current_node(NULL) {} - - //! Copy constructor - graph_iterator(const graph_iterator& other) : - my_graph(other.my_graph), current_node(other.current_node) - {} - - //! Assignment - graph_iterator& operator=(const graph_iterator& other) { - if (this != &other) { - my_graph = other.my_graph; - current_node = other.current_node; - } - return *this; - } - - //! Dereference - reference operator*() const; - - //! Dereference - pointer operator->() const; - - //! Equality - bool operator==(const graph_iterator& other) const { - return ((my_graph == other.my_graph) && (current_node == other.current_node)); - } - - //! Inequality - bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } - - //! Pre-increment - graph_iterator& operator++() { - internal_forward(); - return *this; - } - - //! Post-increment - graph_iterator operator++(int) { - graph_iterator result = *this; - operator++(); - return result; - } - -private: - // the graph over which we are iterating - GraphContainerType *my_graph; - // pointer into my_graph's my_nodes list - pointer current_node; - - //! Private initializing constructor for begin() and end() iterators - graph_iterator(GraphContainerType *g, bool begin); - void internal_forward(); -}; // class graph_iterator - -//! The graph class -/** This class serves as a handle to the graph */ -class graph : tbb::internal::no_copy, public graph_proxy { - friend class graph_node; - - template< typename Body > - class run_task : public task { - public: - run_task( Body& body ) : my_body(body) {} - task *execute() __TBB_override { - my_body(); - return NULL; - } - private: - Body my_body; - }; - - template< typename Receiver, typename Body > - class run_and_put_task : public task { - public: - run_and_put_task( Receiver &r, Body& body ) : my_receiver(r), my_body(body) {} - task *execute() __TBB_override { - task *res = my_receiver.try_put_task( my_body() ); - if (res == SUCCESSFULLY_ENQUEUED) res = NULL; - return res; - } - private: - Receiver &my_receiver; - Body my_body; - }; - typedef std::list task_list_type; - - class wait_functor { - task* graph_root_task; - public: - wait_functor( task* t ) : graph_root_task(t) {} - void operator()() const { graph_root_task->wait_for_all(); } - }; - - void prepare_task_arena( bool reinit = false ) { - if (reinit) { - __TBB_ASSERT( my_task_arena, "task arena is NULL"); - my_task_arena->terminate(); - my_task_arena->initialize(tbb::task_arena::attach()); - } else { - __TBB_ASSERT(my_task_arena == NULL, "task arena is not NULL"); - my_task_arena = new tbb::task_arena(tbb::task_arena::attach()); - } - if (!my_task_arena->is_active()) // failed to attach - my_task_arena->initialize(); // create a new, default-initialized arena - __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); - } - -public: - //! Constructs a graph with isolated task_group_context - graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = true; - cancelled = false; - caught_exception = false; - my_context = new task_group_context(); - my_root_task = ( new ( task::allocate_root(*my_context) ) empty_task ); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph( this ); - my_is_active = true; - } - - //! Constructs a graph with use_this_context as context - explicit graph(task_group_context& use_this_context) : - my_context(&use_this_context), my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = false; - my_root_task = ( new ( task::allocate_root(*my_context) ) empty_task ); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph( this ); - my_is_active = true; - } - - //! Destroys the graph. - /** Calls wait_for_all, then destroys the root task and context. */ - ~graph() { - wait_for_all(); - my_root_task->set_ref_count(0); - task::destroy( *my_root_task ); - if (own_context) delete my_context; - delete my_task_arena; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { - tbb::internal::fgt_graph_desc( this, name ); - } -#endif - - //! Used to register that an external entity may still interact with the graph. - /** The graph will not return from wait_for_all until a matching number of decrement_wait_count calls - is made. */ - void increment_wait_count() { - if (my_root_task) - my_root_task->increment_ref_count(); - } - - //! Deregisters an external entity that may have interacted with the graph. - /** The graph will not return from wait_for_all until all the number of decrement_wait_count calls - matches the number of increment_wait_count calls. */ - void decrement_wait_count() { - if (my_root_task) - my_root_task->decrement_ref_count(); - } - - void reserve_wait() __TBB_override { - increment_wait_count(); - } - - void release_wait() __TBB_override { - decrement_wait_count(); - } - - //! Spawns a task that runs a body and puts its output to a specific receiver - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Receiver, typename Body > - void run( Receiver &r, Body body ) { - if(is_active()) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *root_task() ) ) - run_and_put_task< Receiver, Body >( r, body )) ); - } - } - - //! Spawns a task that runs a function object - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Body > - void run( Body body ) { - if(is_active()) { - FLOW_SPAWN( * new ( task::allocate_additional_child_of( *root_task() ) ) run_task< Body >( body ) ); - } - } - - //! Wait until graph is idle and decrement_wait_count calls equals increment_wait_count calls. - /** The waiting thread will go off and steal work while it is block in the wait_for_all. */ - void wait_for_all() { - cancelled = false; - caught_exception = false; - if (my_root_task) { -#if TBB_USE_EXCEPTIONS - try { -#endif - my_task_arena->execute(wait_functor(my_root_task)); - cancelled = my_context->is_group_execution_cancelled(); -#if TBB_USE_EXCEPTIONS - } - catch(...) { - my_root_task->set_ref_count(1); - my_context->reset(); - caught_exception = true; - cancelled = true; - throw; - } +#if __TBB_PREVIEW_ASYNC_MSG +#include "internal/_flow_graph_async_msg_impl.h" #endif - // TODO: the "if" condition below is just a work-around to support the concurrent wait - // mode. The cancellation and exception mechanisms are still broken in this mode. - // Consider using task group not to re-implement the same functionality. - if ( !(my_context->traits() & task_group_context::concurrent_wait) ) { - my_context->reset(); // consistent with behavior in catch() - my_root_task->set_ref_count(1); - } - } - } - - //! Returns the root task of the graph - task * root_task() { - return my_root_task; - } - - void set_active(bool a = true) { - my_is_active = a; - } - - bool is_active() { - return my_is_active; - } - - void add_task_to_reset_list(task *tp) { - my_reset_task_list.push_back(tp); - } - - // ITERATORS - template - friend class graph_iterator; - - // Graph iterator typedefs - typedef graph_iterator iterator; - typedef graph_iterator const_iterator; - - // Graph iterator constructors - //! start iterator - iterator begin() { return iterator(this, true); } - //! end iterator - iterator end() { return iterator(this, false); } - //! start const iterator - const_iterator begin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator end() const { return const_iterator(this, false); } - //! start const iterator - const_iterator cbegin() const { return const_iterator(this, true); } - //! end const iterator - const_iterator cend() const { return const_iterator(this, false); } - - //! return status of graph execution - bool is_cancelled() { return cancelled; } - bool exception_thrown() { return caught_exception; } - - // thread-unsafe state reset. - void reset(reset_flags f = rf_reset_protocol); - -private: - task *my_root_task; - task_group_context *my_context; - bool own_context; - bool cancelled; - bool caught_exception; - bool my_is_active; - task_list_type my_reset_task_list; - - graph_node *my_nodes, *my_nodes_last; - - spin_mutex nodelist_mutex; - void register_node(graph_node *n); - void remove_node(graph_node *n); - - template < typename Input, typename Output, typename Policy, typename Allocator > - friend class async_node; - task_arena* my_task_arena; -}; // class graph +using namespace internal::graph_policy_namespace; template graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(NULL) @@ -954,34 +739,50 @@ void graph_iterator::internal_forward() { if (current_node) current_node = current_node->next; } -//! The base of all graph nodes. -class graph_node : tbb::internal::no_copy { - friend class graph; - template - friend class graph_iterator; -protected: - graph& my_graph; - graph_node *next, *prev; -public: - explicit graph_node(graph& g) : my_graph(g) { - my_graph.register_node(this); - } - virtual ~graph_node() { - my_graph.remove_node(this); - } +//! Constructs a graph with isolated task_group_context +inline graph::graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { + prepare_task_arena(); + own_context = true; + cancelled = false; + caught_exception = false; + my_context = new task_group_context(tbb::internal::FLOW_TASKS); + my_root_task = (new (task::allocate_root(*my_context)) empty_task); + my_root_task->set_ref_count(1); + tbb::internal::fgt_graph(this); + my_is_active = true; +} -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - virtual void set_name( const char *name ) = 0; -#endif +inline graph::graph(task_group_context& use_this_context) : + my_context(&use_this_context), my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { + prepare_task_arena(); + own_context = false; + my_root_task = (new (task::allocate_root(*my_context)) empty_task); + my_root_task->set_ref_count(1); + tbb::internal::fgt_graph(this); + my_is_active = true; +} -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - virtual void extract( ) = 0; -#endif +inline graph::~graph() { + wait_for_all(); + my_root_task->set_ref_count(0); + tbb::task::destroy(*my_root_task); + if (own_context) delete my_context; + delete my_task_arena; +} -protected: - // performs the reset on an individual node. - virtual void reset_node(reset_flags f=rf_reset_protocol) = 0; -}; // class graph_node +inline void graph::reserve_wait() { + if (my_root_task) { + my_root_task->increment_ref_count(); + tbb::internal::fgt_reserve_wait(this); + } +} + +inline void graph::release_wait() { + if (my_root_task) { + tbb::internal::fgt_release_wait(this); + my_root_task->decrement_ref_count(); + } +} inline void graph::register_node(graph_node *n) { n->next = NULL; @@ -1008,7 +809,8 @@ inline void graph::remove_node(graph_node *n) { inline void graph::reset( reset_flags f ) { // reset context - set_active(false); + internal::deactivate_graph(*this); + if(my_context) my_context->reset(); cancelled = false; caught_exception = false; @@ -1020,14 +822,40 @@ inline void graph::reset( reset_flags f ) { // Reattach the arena. Might be useful to run the graph in a particular task_arena // while not limiting graph lifetime to a single task_arena::execute() call. prepare_task_arena( /*reinit=*/true ); - set_active(true); + internal::activate_graph(*this); // now spawn the tasks necessary to start the graph for(task_list_type::iterator rti = my_reset_task_list.begin(); rti != my_reset_task_list.end(); ++rti) { - FLOW_SPAWN(*(*rti)); + internal::spawn_in_graph_arena(*this, *(*rti)); } my_reset_task_list.clear(); } +inline graph::iterator graph::begin() { return iterator(this, true); } + +inline graph::iterator graph::end() { return iterator(this, false); } + +inline graph::const_iterator graph::begin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::end() const { return const_iterator(this, false); } + +inline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::cend() const { return const_iterator(this, false); } + +#if TBB_PREVIEW_FLOW_GRAPH_TRACE +inline void graph::set_name(const char *name) { + tbb::internal::fgt_graph_desc(this, name); +} +#endif + +inline graph_node::graph_node(graph& g) : my_graph(g) { + my_graph.register_node(this); +} + +inline graph_node::~graph_node() { + my_graph.remove_node(this); +} + #include "internal/_flow_graph_node_impl.h" //! An executable node that acts as a source, i.e. it has no predecessors @@ -1043,7 +871,7 @@ class source_node : public graph_node, public sender< Output > { //Source node has no input type typedef null_type input_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -1098,7 +926,7 @@ class source_node : public graph_node, public sender< Output > { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -1121,7 +949,7 @@ class source_node : public graph_node, public sender< Output > { spin_mutex::scoped_lock l(my_mutex); my_successors.copy_successors(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Request an item from the node bool try_get( output_type &v ) __TBB_override { @@ -1183,7 +1011,7 @@ class source_node : public graph_node, public sender< Output > { void activate() { spin_mutex::scoped_lock lock(my_mutex); my_active = true; - if ( !my_successors.empty() ) + if (!my_successors.empty()) spawn_put(); } @@ -1193,7 +1021,7 @@ class source_node : public graph_node, public sender< Output > { return dynamic_cast< internal::source_body_leaf & >(body_ref).get_body(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) __TBB_override { my_successors.built_successors().sender_extract(*this); // removes "my_owner" == this from each successor my_active = init_my_active; @@ -1218,7 +1046,7 @@ class source_node : public graph_node, public sender< Output > { my_body = tmp; } if(my_active) - this->my_graph.add_task_to_reset_list(create_put_task()); + internal::add_task_to_graph_reset_list(this->my_graph, create_put_task()); } private: @@ -1266,8 +1094,8 @@ class source_node : public graph_node, public sender< Output > { //! Spawns a task that applies the body void spawn_put( ) { - if(this->my_graph.is_active()) { - FLOW_SPAWN( *create_put_task()); + if(internal::is_graph_active(this->my_graph)) { + internal::spawn_in_graph_arena(this->my_graph, *create_put_task()); } } @@ -1287,41 +1115,32 @@ class source_node : public graph_node, public sender< Output > { } }; // class source_node -template -struct allocate_buffer { - static const bool value = false; -}; - -template<> -struct allocate_buffer { - static const bool value = true; -}; - //! Implements a function node that supports Input -> Output template < typename Input, typename Output = continue_msg, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class function_node : public graph_node, public internal::function_input, public internal::function_output { +class function_node : public graph_node, public internal::function_input, public internal::function_output { public: typedef Input input_type; typedef Output output_type; - typedef internal::function_input fInput_type; + typedef internal::function_input input_impl_type; typedef internal::function_input_queue input_queue_type; typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; + typedef typename input_impl_type::predecessor_type predecessor_type; typedef typename fOutput_type::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename fInput_type::predecessor_list_type predecessor_list_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename input_impl_type::predecessor_list_type predecessor_list_type; typedef typename fOutput_type::successor_list_type successor_list_type; #endif - using fInput_type::my_predecessors; + using input_impl_type::my_predecessors; //! Constructor // input_queue_type is allocated here, but destroyed in the function_input_base. // TODO: pass the graph_buffer_policy to the function_input_base so it can all // be done in one place. This would be an interface-breaking change. template< typename Body > - function_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), fInput_type(g, concurrency, body, allocate_buffer::value ? - new input_queue_type( ) : NULL ) { + function_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), input_impl_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); } @@ -1329,7 +1148,7 @@ class function_node : public graph_node, public internal::function_input::value ? new input_queue_type : NULL), + input_impl_type(src), fOutput_type() { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); @@ -1341,7 +1160,7 @@ class function_node : public graph_node, public internal::function_input friend class run_and_put_task; template friend class internal::broadcast_cache; template friend class internal::round_robin_cache; - using fInput_type::try_put_task; + using input_impl_type::try_put_task; internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_function_input(f); + input_impl_type::reset_function_input(f); // TODO: use clear() instead. if(f & rf_clear_edges) { successors().clear(); @@ -1382,6 +1201,7 @@ class multifunction_node : internal::multifunction_output, // wrap this around each element Output // the tuple providing the types >::type, + Policy, Allocator > { protected: @@ -1390,22 +1210,26 @@ class multifunction_node : typedef Input input_type; typedef null_type output_type; typedef typename internal::wrap_tuple_elements::type output_ports_type; - typedef internal::multifunction_input fInput_type; + typedef internal::multifunction_input input_impl_type; typedef internal::function_input_queue input_queue_type; private: - typedef typename internal::multifunction_input base_type; - using fInput_type::my_predecessors; + typedef typename internal::multifunction_input base_type; + using input_impl_type::my_predecessors; public: template - multifunction_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), base_type(g,concurrency, body, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); + multifunction_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), base_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { + tbb::internal::fgt_multioutput_node_with_body( + tbb::internal::FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); } multifunction_node( const multifunction_node &other) : - graph_node(other.my_graph), base_type(other, allocate_buffer::value ? new input_queue_type : NULL) { + graph_node(other.my_graph), base_type(other) { tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, &this->my_graph, static_cast *>(this), this->output_ports(), this->my_body ); @@ -1417,7 +1241,7 @@ class multifunction_node : } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) __TBB_override { my_predecessors.built_predecessors().receiver_extract(*this); base_type::extract(); @@ -1437,7 +1261,7 @@ class split_node : public graph_node, public receiver { public: typedef TupleType input_type; typedef Allocator allocator_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename base_type::predecessor_type predecessor_type; typedef typename base_type::predecessor_list_type predecessor_list_type; typedef internal::predecessor_cache predecessor_cache_type; @@ -1471,11 +1295,9 @@ class split_node : public graph_node, public receiver { protected: task *try_put_task(const TupleType& t) __TBB_override { - // Sending split messages in parallel is not justified, as overheads would prevail - internal::emit_element::emit_this(t, output_ports()); - - //we do not have successors here.So we just tell the task is successful. - return SUCCESSFULLY_ENQUEUED; + // Sending split messages in parallel is not justified, as overheads would prevail. + // Also, we do not have successors here. So we just tell the task returned here is successful. + return internal::emit_element::emit_this(this->my_graph, t, output_ports()); } void reset_node(reset_flags f) __TBB_override { if (f & rf_clear_edges) @@ -1484,8 +1306,10 @@ class split_node : public graph_node, public receiver { __TBB_ASSERT(!(f & rf_clear_edges) || internal::clear_element::this_empty(my_output_ports), "split_node reset failed"); } void reset_receiver(reset_flags /*f*/) __TBB_override {} - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + graph& graph_reference() __TBB_override { + return my_graph; + } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION private: //! split_node doesn't use this "predecessors" functionality; so, we have "dummies" here; void extract() __TBB_override {} @@ -1503,37 +1327,42 @@ class split_node : public graph_node, public receiver { //! dummy member built_predecessors_type my_predessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ private: output_ports_type my_output_ports; }; //! Implements an executable node that supports continue_msg -> Output -template -class continue_node : public graph_node, public internal::continue_input, public internal::function_output { +template > +class continue_node : public graph_node, public internal::continue_input, + public internal::function_output { public: typedef continue_msg input_type; typedef Output output_type; - typedef internal::continue_input fInput_type; + typedef internal::continue_input input_impl_type; typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; + typedef typename input_impl_type::predecessor_type predecessor_type; typedef typename fOutput_type::successor_type successor_type; //! Constructor for executable node with continue_msg -> Output template - continue_node( graph &g, Body body ) : - graph_node(g), internal::continue_input( g, body ) { + continue_node( + graph &g, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), input_impl_type( g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority) ) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); } - //! Constructor for executable node with continue_msg -> Output template - continue_node( graph &g, int number_of_predecessors, Body body ) : - graph_node(g), internal::continue_input( g, number_of_predecessors, body ) { + continue_node( + graph &g, int number_of_predecessors, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g) + , input_impl_type(g, number_of_predecessors, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); @@ -1541,7 +1370,7 @@ class continue_node : public graph_node, public internal::continue_input //! Copy constructor continue_node( const continue_node& src ) : - graph_node(src.my_graph), internal::continue_input(src), + graph_node(src.my_graph), input_impl_type(src), internal::function_output() { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), @@ -1554,9 +1383,9 @@ class continue_node : public graph_node, public internal::continue_input } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { - fInput_type::my_built_predecessors.receiver_extract(*this); + input_impl_type::my_built_predecessors.receiver_extract(*this); successors().built_successors().sender_extract(*this); } #endif @@ -1565,252 +1394,48 @@ class continue_node : public graph_node, public internal::continue_input template< typename R, typename B > friend class run_and_put_task; template friend class internal::broadcast_cache; template friend class internal::round_robin_cache; - using fInput_type::try_put_task; + using input_impl_type::try_put_task; internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_receiver(f); + input_impl_type::reset_receiver(f); if(f & rf_clear_edges)successors().clear(); __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); } }; // continue_node -template< typename T > -class overwrite_node : public graph_node, public receiver, public sender { +//! Forwards messages of type T to all successors +template +class broadcast_node : public graph_node, public receiver, public sender { public: typedef T input_type; typedef T output_type; typedef typename receiver::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename sender::successor_list_type successor_list_type; #endif +private: + internal::broadcast_cache my_successors; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + internal::edge_container my_built_predecessors; + spin_mutex pred_mutex; // serialize accesses on edge_container +#endif +public: - explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { + explicit broadcast_node(graph& g) : graph_node(g) { my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this) ); } - // Copy constructor; doesn't take anything from src; default won't work - overwrite_node( const overwrite_node& src ) : - graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) + // Copy constructor + broadcast_node( const broadcast_node& src ) : + graph_node(src.my_graph), receiver(), sender() { my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - ~overwrite_node() {} - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - bool register_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if (my_buffer_is_valid && this->my_graph.is_active()) { - // We have a valid value that must be forwarded immediately. - if ( s.try_put( my_buffer ) || !s.register_predecessor( *this ) ) { - // We add the successor: it accepted our put or it rejected it but won't let us become a predecessor - my_successors.register_successor( s ); - } else { - // We don't add the successor: it rejected our put and we became its predecessor instead - return false; - } - } else { - // No valid value yet, just add as successor - my_successors.register_successor( s ); - } - return true; - } - - bool remove_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.remove_successor(s); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_add_built_successor(s); - } - - void internal_delete_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_delete_built_successor(s); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_buffer_is_valid = false; - built_successors().sender_extract(*this); - built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - bool try_get( input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if ( my_buffer_is_valid ) { - v = my_buffer; - return true; - } - return false; - } - - bool is_valid() { - spin_mutex::scoped_lock l( my_mutex ); - return my_buffer_is_valid; - } - - void clear() { - spin_mutex::scoped_lock l( my_mutex ); - my_buffer_is_valid = false; - } - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task * try_put_task( const input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return try_put_task_impl(v); - } - - task * try_put_task_impl(const input_type &v) { - my_buffer = v; - my_buffer_is_valid = true; - task * rtask = my_successors.try_put_task(v); - if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; - return rtask; - } - - spin_mutex my_mutex; - internal::broadcast_cache< input_type, null_rw_mutex > my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - input_type my_buffer; - bool my_buffer_is_valid; - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node( reset_flags f) __TBB_override { - my_buffer_is_valid = false; - if (f&rf_clear_edges) { - my_successors.clear(); - } - } -}; // overwrite_node - -template< typename T > -class write_once_node : public overwrite_node { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit write_once_node(graph& g) : overwrite_node(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor: call base class copy constructor - write_once_node( const write_once_node& src ) : overwrite_node(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T &v ) __TBB_override { - spin_mutex::scoped_lock l( this->my_mutex ); - return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); - } -}; - -//! Forwards messages of type T to all successors -template -class broadcast_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -private: - internal::broadcast_cache my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; - spin_mutex pred_mutex; // serialize accesses on edge_container -#endif -public: - - explicit broadcast_node(graph& g) : graph_node(g) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - // Copy constructor - broadcast_node( const broadcast_node& src ) : - graph_node(src.my_graph), receiver(), sender() - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, + tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this) ); } @@ -1832,7 +1457,7 @@ class broadcast_node : public graph_node, public receiver, public sender { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -1855,7 +1480,7 @@ class broadcast_node : public graph_node, public receiver, public sender { typedef typename receiver::built_predecessors_type built_predecessors_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } void internal_add_built_predecessor( predecessor_type &p) __TBB_override { spin_mutex::scoped_lock l(pred_mutex); @@ -1881,7 +1506,7 @@ class broadcast_node : public graph_node, public receiver, public sender { my_built_predecessors.receiver_extract(*this); my_successors.built_successors().sender_extract(*this); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template< typename R, typename B > friend class run_and_put_task; @@ -1894,12 +1519,16 @@ class broadcast_node : public graph_node, public receiver, public sender { return new_task; } + graph& graph_reference() __TBB_override { + return my_graph; + } + void reset_receiver(reset_flags /*f*/) __TBB_override {} void reset_node(reset_flags f) __TBB_override { if (f&rf_clear_edges) { my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_predecessors.clear(); #endif } @@ -1916,7 +1545,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; typedef buffer_node class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -1924,14 +1553,14 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION internal::edge_container my_built_predecessors; #endif friend class internal::forward_task_bypass< buffer_node< T, A > >; enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, add_blt_pred, del_blt_pred, blt_succ_cnt, blt_pred_cnt, @@ -1943,7 +1572,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer { public: char type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION task * ltask; union { input_type *elem; @@ -1960,7 +1589,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer(&e)) #else , elem(const_cast(&e)) , ltask(NULL) @@ -1996,7 +1625,7 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferorder(); if (try_forwarding && !forwarder_busy) { - if(this->my_graph.is_active()) { + if(internal::is_graph_active(this->my_graph)) { forwarder_busy = true; task *new_task = new(task::allocate_additional_child_of(*(this->my_graph.root_task()))) internal:: forward_task_bypass < buffer_node >(*this); // tmp should point to the last item handled by the aggregator. This is the operation // the handling thread enqueued. So modifying that record will be okay. + // workaround for icc bug tbb::task *z = tmp->ltask; - tmp->ltask = combine_tasks(z, new_task); // in case the op generated a task + graph &g = this->my_graph; + tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task } } } // handle_operations @@ -2033,7 +1664,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffermy_graph; + last_task = combine_tasks(g, last_task, xtask); } while (op_data.status ==internal::SUCCEEDED); return last_task; } @@ -2065,7 +1699,7 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferstatus, internal::SUCCEEDED); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -2082,7 +1716,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer::built_predecessors_type built_predecessors_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } virtual void internal_add_built_pred(buffer_operation *op) { my_built_predecessors.add_edge(*(op->p)); @@ -2114,7 +1748,7 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferstatus, internal::SUCCEEDED); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ private: void order() {} @@ -2126,7 +1760,9 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferback()); if (new_task) { - last_task = combine_tasks(last_task, new_task); + // workaround for icc bug + graph& g = this->my_graph; + last_task = combine_tasks(g, last_task, new_task); this->destroy_back(); } } @@ -2236,7 +1872,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer { void try_put_and_add_task(task*& last_task) { task *new_task = this->my_successors.try_put_task(this->front()); if (new_task) { - last_task = combine_tasks(last_task, new_task); + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); this->destroy_front(); } } @@ -2652,7 +2294,9 @@ class priority_queue_node : public buffer_node { void try_put_and_add_task(task*& last_task) { task * new_task = this->my_successors.try_put_task(this->prio()); if (new_task) { - last_task = combine_tasks(last_task, new_task); + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); prio_pop(); } } @@ -2750,18 +2394,25 @@ class priority_queue_node : public buffer_node { } }; // priority_queue_node +} // interfaceX + +namespace interface11 { + +using namespace interface10; +namespace internal = interface10::internal; + //! Forwards messages only if the threshold has not been reached /** This node forwards items until its threshold is reached. It contains no buffering. If the downstream node rejects, the message is dropped. */ -template< typename T > +template< typename T, typename DecrementType=continue_msg > class limiter_node : public graph_node, public receiver< T >, public sender< T > { public: typedef T input_type; typedef T output_type; typedef typename receiver::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename sender::built_successors_type built_successors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; @@ -2776,12 +2427,12 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; spin_mutex my_mutex; internal::broadcast_cache< T > my_successors; - int init_decrement_predecessors; + __TBB_DEPRECATED_LIMITER_EXPR( int init_decrement_predecessors; ) - friend class internal::forward_task_bypass< limiter_node >; + friend class internal::forward_task_bypass< limiter_node >; // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node >; + friend class internal::decrementer< limiter_node, DecrementType >; bool check_conditions() { // always called under lock return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); @@ -2812,10 +2463,10 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > --my_tries; my_predecessors.try_consume(); if ( check_conditions() ) { - if ( this->my_graph.is_active() ) { + if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - FLOW_SPAWN (*rtask); + internal::forward_task_bypass< limiter_node >( *this ); + internal::spawn_in_graph_arena(graph_reference(), *rtask); } } } @@ -2830,9 +2481,9 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > --my_tries; if (reserved) my_predecessors.try_release(); if ( check_conditions() ) { - if ( this->my_graph.is_active() ) { + if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); __TBB_ASSERT(!rval, "Have two tasks to handle"); return rtask; } @@ -2846,45 +2497,59 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return; } - task * decrement_counter() { + task* decrement_counter( long long delta ) { { spin_mutex::scoped_lock lock(my_mutex); - if(my_count) --my_count; + if( delta > 0 && size_t(delta) > my_count ) + my_count = 0; + else if( delta < 0 && size_t(delta) > my_threshold - my_count ) + my_count = my_threshold; + else + my_count -= size_t(delta); // absolute value of delta is sufficiently small } return forward_task(); } -public: - //! The internal receiver< continue_msg > that decrements the count - internal::decrementer< limiter_node > decrement; - - //! Constructor - limiter_node(graph &g, size_t threshold, int num_decrement_predecessors=0) : - graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors) - { + void initialize() { my_predecessors.set_owner(this); my_successors.set_owner(this); decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + tbb::internal::fgt_node( + tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(&decrement), + static_cast *>(this) + ); + } +public: + //! The internal receiver< DecrementType > that decrements the count + internal::decrementer< limiter_node, DecrementType > decrement; + +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), + "Deprecated interface of the limiter node can be used only in conjunction " + "with continue_msg as the type of DecrementType template parameter." ); +#endif // Check for incompatible interface + + //! Constructor + limiter_node(graph &g, + __TBB_DEPRECATED_LIMITER_ARG2(size_t threshold, int num_decrement_predecessors=0)) + : graph_node(g), my_threshold(threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(num_decrement_predecessors), + decrement(num_decrement_predecessors)) { + initialize(); } //! Copy constructor limiter_node( const limiter_node& src ) : graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), my_tries(0), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + my_threshold(src.my_threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(src.init_decrement_predecessors), + decrement(src.init_decrement_predecessors)) { + initialize(); } #if TBB_PREVIEW_FLOW_GRAPH_TRACE @@ -2900,9 +2565,10 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > my_successors.register_successor(r); //spawn a forward task if this is the only successor if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { - if ( this->my_graph.is_active() ) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ) ) ); + if ( internal::is_graph_active(this->my_graph) ) { + task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) + internal::forward_task_bypass < limiter_node >( *this ); + internal::spawn_in_graph_arena(graph_reference(), *task); } } return true; @@ -2916,7 +2582,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } @@ -2954,15 +2620,16 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > my_predecessors.built_predecessors().receiver_extract(*this); decrement.built_predecessors().receiver_extract(decrement); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Adds src to the list of cached predecessors. bool register_predecessor( predecessor_type &src ) __TBB_override { spin_mutex::scoped_lock lock(my_mutex); my_predecessors.add( src ); - if ( my_count + my_tries < my_threshold && !my_successors.empty() && this->my_graph.is_active() ) { - FLOW_SPAWN( (* new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ) ) ); + if ( my_count + my_tries < my_threshold && !my_successors.empty() && internal::is_graph_active(this->my_graph) ) { + task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) + internal::forward_task_bypass < limiter_node >( *this ); + internal::spawn_in_graph_arena(graph_reference(), *task); } return true; } @@ -2993,9 +2660,9 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( !rtask ) { // try_put_task failed. spin_mutex::scoped_lock lock(my_mutex); --my_tries; - if ( check_conditions() && this->my_graph.is_active() ) { + if (check_conditions() && internal::is_graph_active(this->my_graph)) { rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); } } else { @@ -3006,6 +2673,8 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return rtask; } + graph& graph_reference() __TBB_override { return my_graph; } + void reset_receiver(reset_flags /*f*/) __TBB_override { __TBB_ASSERT(false,NULL); // should never be called } @@ -3023,6 +2692,9 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > decrement.reset_receiver(f); } }; // limiter_node +} // namespace interfaceX + +namespace interface10 { #include "internal/_flow_graph_join_impl.h" @@ -3207,7 +2879,7 @@ class indexer_node : public internal::unfolded_indexer_node > { } #if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { + void set_name( const char *name ) __TBB_override { tbb::internal::fgt_node_desc( this, name ); } #endif @@ -3232,7 +2904,7 @@ class indexer_node : public internal::unfolded_indexer_node : public internal::unfolded_indexer_node inline void internal_make_edge( sender &p, receiver &s ) { #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION s.internal_add_built_predecessor(p); p.internal_add_built_successor(s); #endif @@ -3522,7 +3194,7 @@ template< typename T > inline void internal_remove_edge( sender &p, receiver &s ) { #endif p.remove_successor( s ); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // TODO: should we try to remove p from the predecessor list of s, in case the edge is reversed? p.internal_delete_built_successor(s); s.internal_delete_built_predecessor(p); @@ -3577,7 +3249,7 @@ inline void remove_edge( sender& output, V& input) { } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template template< typename S > void internal::edge_container::sender_extract( S &s ) { @@ -3595,7 +3267,7 @@ void internal::edge_container::receiver_extract( R &r ) { remove_edge(**i, r); } } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Returns a copy of the body from a function or continue node template< typename Body, typename Node > @@ -3616,11 +3288,8 @@ class composite_node , tbb::flow::tuple&... > output_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - input_ports_type *my_input_ports; - output_ports_type *my_output_ports; + std::unique_ptr my_input_ports; + std::unique_ptr my_output_ports; static const size_t NUM_INPUTS = sizeof...(InputTypes); static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); @@ -3630,37 +3299,32 @@ class composite_node , tbb::flow::tuplemy_graph ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } #else - composite_node( graph &g) : graph_node(g), my_input_ports(NULL), my_output_ports(NULL) {} + composite_node( graph &g ) : graph_node(g) { + tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); + } #endif - template - void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_input_ports = new input_ports_type(std::forward(input_ports_tuple)); - my_output_ports = new output_ports_type(std::forward(output_ports_tuple)); + template + void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { + __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); + __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); + my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); + my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_helper::register_port( this, input_ports_tuple); - tbb::internal::fgt_internal_output_helper::register_port( this, output_ports_tuple); -#endif - } + tbb::internal::fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + tbb::internal::fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template< typename... NodeTypes > void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template< typename... NodeTypes > void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) { } - template void add_visible_nodes(Nodes&...) { } -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3668,22 +3332,17 @@ class composite_node , tbb::flow::tuple, tbb::flow::tuple<> > : pu typedef tbb::flow::tuple< receiver&... > input_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - input_ports_type *my_input_ports; + std::unique_ptr my_input_ports; static const size_t NUM_INPUTS = sizeof...(InputTypes); protected: @@ -3708,35 +3364,30 @@ class composite_node , tbb::flow::tuple<> > : pu public: #if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name), my_input_ports(NULL) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); + composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } #else - composite_node( graph &g) : graph_node(g), my_input_ports(NULL) {} + composite_node( graph &g ) : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + } #endif template void set_external_ports(T&& input_ports_tuple) { __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - my_input_ports = new input_ports_type(std::forward(input_ports_tuple)); + my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_helper::register_port( this, std::forward(input_ports_tuple)); -#endif + tbb::internal::fgt_internal_input_alias_helper::alias_port( this, std::forward(input_ports_tuple)); } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template< typename... NodeTypes > void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template< typename... NodeTypes > void add_nodes( const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3744,16 +3395,12 @@ class composite_node , tbb::flow::tuple<> > : pu } #endif - input_ports_type input_ports() { + input_ports_type& input_ports() { __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); return *my_input_ports; } - virtual ~composite_node() { - if(my_input_ports) delete my_input_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); } @@ -3768,10 +3415,7 @@ class composite_node , tbb::flow::tuple > : p typedef tbb::flow::tuple< sender&... > output_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif - output_ports_type *my_output_ports; + std::unique_ptr my_output_ports; static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); protected: @@ -3779,35 +3423,30 @@ class composite_node , tbb::flow::tuple > : p public: #if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name), my_output_ports(NULL) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); + composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } #else - composite_node( graph &g) : graph_node(g), my_output_ports(NULL) {} + composite_node( graph &g ) : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + } #endif template void set_external_ports(T&& output_ports_tuple) { __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_output_ports = new output_ports_type(std::forward(output_ports_tuple)); + my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_output_helper::register_port( this, std::forward(output_ports_tuple)); -#endif + tbb::internal::fgt_internal_output_alias_helper::alias_port( this, std::forward(output_ports_tuple)); } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3815,16 +3454,12 @@ class composite_node , tbb::flow::tuple > : p } #endif - output_ports_type output_ports() { + output_ports_type& output_ports() { __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); return *my_output_ports; } - virtual ~composite_node() { - if(my_output_ports) delete my_output_ports; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); } @@ -3856,7 +3491,7 @@ class async_body: public async_body_base { typedef async_body_base base_type; typedef Gateway gateway_type; - async_body(const Body &body, gateway_type *gateway) + async_body(const Body &body, gateway_type *gateway) : base_type(gateway), my_body(body) { } void operator()( const Input &v, Ports & ) { @@ -3872,10 +3507,12 @@ class async_body: public async_body_base { } //! Implements async node -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > +template < typename Input, typename Output, + typename Policy = queueing_lightweight, + typename Allocator=cache_aligned_allocator > class async_node : public multifunction_node< Input, tuple< Output >, Policy, Allocator >, public sender< Output > { typedef multifunction_node< Input, tuple< Output >, Policy, Allocator > base_type; - typedef typename internal::multifunction_input mfn_input_type; + typedef typename internal::multifunction_input mfn_input_type; public: typedef Input input_type; @@ -3885,11 +3522,13 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al typedef typename sender::successor_type successor_type; typedef receiver_gateway gateway_type; typedef internal::async_body_base async_body_base_type; + typedef typename base_type::output_ports_type output_ports_type; private: struct try_put_functor { typedef internal::multifunction_output output_port_type; output_port_type *port; + // TODO: pass value by copy since we do not want to block asynchronous thread. const Output *value; bool result; try_put_functor(output_port_type &p, const Output &v) : port(&p), value(&v), result(false) { } @@ -3902,8 +3541,8 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al public: receiver_gateway_impl(async_node* node): my_node(node) {} void reserve_wait() __TBB_override { - my_node->my_graph.reserve_wait(); tbb::internal::fgt_async_reserve(static_cast(my_node), &my_node->my_graph); + my_node->my_graph.reserve_wait(); } void release_wait() __TBB_override { @@ -3926,29 +3565,43 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al //! Implements gateway_type::try_put for an external activity to submit a message to FG bool try_put_impl(const Output &i) { internal::multifunction_output &port_0 = internal::output_port<0>(*this); + internal::broadcast_cache& port_successors = port_0.successors(); tbb::internal::fgt_async_try_put_begin(this, &port_0); - __TBB_ASSERT(this->my_graph.my_task_arena && this->my_graph.my_task_arena->is_active(), NULL); - try_put_functor tpf(port_0, i); - this->my_graph.my_task_arena->execute(tpf); + task_list tasks; + bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); + __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), + "Return status is inconsistent with the method operation." ); + + while( !tasks.empty() ) { + internal::enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); + } tbb::internal::fgt_async_try_put_end(this, &port_0); - return tpf.result; + return is_at_least_one_put_successful; } public: template - async_node( graph &g, size_t concurrency, Body body ) : - base_type( g, concurrency, internal::async_body(body, &my_gateway) ), my_gateway(self()) { - tbb::internal::fgt_multioutput_node<1>( tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, - static_cast *>(this), - this->output_ports() ); + async_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : base_type( + g, concurrency, + internal::async_body + (body, &my_gateway) __TBB_FLOW_GRAPH_PRIORITY_ARG0(priority) ), my_gateway(self()) { + tbb::internal::fgt_multioutput_node_with_body<1>( + tbb::internal::FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); } async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); - tbb::internal::fgt_multioutput_node<1>( tbb::internal::FLOW_ASYNC_NODE, &this->my_graph, static_cast *>(this), this->output_ports() ); + tbb::internal::fgt_multioutput_node_with_body<1>( tbb::internal::FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body ); } gateway_type& gateway() { @@ -3957,7 +3610,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); + tbb::internal::fgt_multioutput_node_desc( this, name ); } #endif @@ -3982,7 +3635,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al return ab.get_body(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef typename internal::edge_container built_successors_type; typedef typename built_successors_type::edge_list_type successor_list_type; @@ -4005,7 +3658,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al size_t successor_count() __TBB_override { return internal::output_port<0>(*this).successor_count(); } -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: @@ -4018,53 +3671,309 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al #include "internal/_flow_graph_streaming_node.h" #endif // __TBB_PREVIEW_STREAMING_NODE -} // interface9 - - using interface9::reset_flags; - using interface9::rf_reset_protocol; - using interface9::rf_reset_bodies; - using interface9::rf_clear_edges; - - using interface9::graph; - using interface9::graph_node; - using interface9::continue_msg; - - using interface9::source_node; - using interface9::function_node; - using interface9::multifunction_node; - using interface9::split_node; - using interface9::internal::output_port; - using interface9::indexer_node; - using interface9::internal::tagged_msg; - using interface9::internal::cast_to; - using interface9::internal::is_a; - using interface9::continue_node; - using interface9::overwrite_node; - using interface9::write_once_node; - using interface9::broadcast_node; - using interface9::buffer_node; - using interface9::queue_node; - using interface9::sequencer_node; - using interface9::priority_queue_node; - using interface9::limiter_node; - using namespace interface9::internal::graph_policy_namespace; - using interface9::join_node; - using interface9::input_port; - using interface9::copy_body; - using interface9::make_edge; - using interface9::remove_edge; - using interface9::internal::tag_value; +} // interfaceX + + +namespace interface10a { + +using namespace interface10; +namespace internal = interface10::internal; + +template< typename T > +class overwrite_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename receiver::built_predecessors_type built_predecessors_type; + typedef typename sender::built_successors_type built_successors_type; + typedef typename receiver::predecessor_list_type predecessor_list_type; + typedef typename sender::successor_list_type successor_list_type; +#endif + + explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { + my_successors.set_owner( this ); + tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + + //! Copy constructor; doesn't take anything from src; default won't work + overwrite_node( const overwrite_node& src ) : + graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) + { + my_successors.set_owner( this ); + tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + + ~overwrite_node() {} + +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + void set_name( const char *name ) __TBB_override { + tbb::internal::fgt_node_desc( this, name ); + } +#endif + + bool register_successor( successor_type &s ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid && internal::is_graph_active( my_graph )) { + // We have a valid value that must be forwarded immediately. + bool ret = s.try_put( my_buffer ); + if ( ret ) { + // We add the successor that accepted our put + my_successors.register_successor( s ); + } else { + // In case of reservation a race between the moment of reservation and register_successor can appear, + // because failed reserve does not mean that register_successor is not ready to put a message immediately. + // We have some sort of infinite loop: reserving node tries to set pull state for the edge, + // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. + task *rtask = new ( task::allocate_additional_child_of( *( my_graph.root_task() ) ) ) + register_predecessor_task( *this, s ); + internal::spawn_in_graph_arena( my_graph, *rtask ); + } + } else { + // No valid value yet, just add as successor + my_successors.register_successor( s ); + } + return true; + } + + bool remove_successor( successor_type &s ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.remove_successor(s); + return true; + } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } + + void internal_add_built_successor( successor_type &s) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.internal_add_built_successor(s); + } + + void internal_delete_built_successor( successor_type &s) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.internal_delete_built_successor(s); + } + + size_t successor_count() __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return my_successors.successor_count(); + } + + void copy_successors(successor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.copy_successors(v); + } + + void internal_add_built_predecessor( predecessor_type &p) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.add_edge(p); + } + + void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.delete_edge(p); + } + + size_t predecessor_count() __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return my_built_predecessors.edge_count(); + } + + void copy_predecessors( predecessor_list_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.copy_edges(v); + } + + void extract() __TBB_override { + my_buffer_is_valid = false; + built_successors().sender_extract(*this); + built_predecessors().receiver_extract(*this); + } + +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + + bool try_get( input_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + if ( my_buffer_is_valid ) { + v = my_buffer; + return true; + } + return false; + } + + //! Reserves an item + bool try_reserve( T &v ) __TBB_override { + return try_get(v); + } + + //! Releases the reserved item + bool try_release() __TBB_override { return true; } + + //! Consumes the reserved item + bool try_consume() __TBB_override { return true; } + + bool is_valid() { + spin_mutex::scoped_lock l( my_mutex ); + return my_buffer_is_valid; + } + + void clear() { + spin_mutex::scoped_lock l( my_mutex ); + my_buffer_is_valid = false; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class internal::broadcast_cache; + template friend class internal::round_robin_cache; + task * try_put_task( const input_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v); + } + + task * try_put_task_impl(const input_type &v) { + my_buffer = v; + my_buffer_is_valid = true; + task * rtask = my_successors.try_put_task(v); + if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; + return rtask; + } + + graph& graph_reference() __TBB_override { + return my_graph; + } + + //! Breaks an infinite loop between the node reservation and register_successor call + struct register_predecessor_task : public graph_task { + + register_predecessor_task(predecessor_type& owner, successor_type& succ) : + o(owner), s(succ) {}; + + tbb::task* execute() __TBB_override { + if (!s.register_predecessor(o)) { + o.register_successor(s); + } + return NULL; + } + + predecessor_type& o; + successor_type& s; + }; + + spin_mutex my_mutex; + internal::broadcast_cache< input_type, null_rw_mutex > my_successors; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + internal::edge_container my_built_predecessors; +#endif + input_type my_buffer; + bool my_buffer_is_valid; + void reset_receiver(reset_flags /*f*/) __TBB_override {} + + void reset_node( reset_flags f) __TBB_override { + my_buffer_is_valid = false; + if (f&rf_clear_edges) { + my_successors.clear(); + } + } +}; // overwrite_node + +template< typename T > +class write_once_node : public overwrite_node { +public: + typedef T input_type; + typedef T output_type; + typedef overwrite_node base_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + explicit write_once_node(graph& g) : base_type(g) { + tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + //! Copy constructor: call base class copy constructor + write_once_node( const write_once_node& src ) : base_type(src) { + tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + void set_name( const char *name ) __TBB_override { + tbb::internal::fgt_node_desc( this, name ); + } +#endif + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class internal::broadcast_cache; + template friend class internal::round_robin_cache; + task *try_put_task( const T &v ) __TBB_override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); + } +}; +} // interfaceX + + using interface10::reset_flags; + using interface10::rf_reset_protocol; + using interface10::rf_reset_bodies; + using interface10::rf_clear_edges; + + using interface10::graph; + using interface10::graph_node; + using interface10::continue_msg; + + using interface10::source_node; + using interface10::function_node; + using interface10::multifunction_node; + using interface10::split_node; + using interface10::internal::output_port; + using interface10::indexer_node; + using interface10::internal::tagged_msg; + using interface10::internal::cast_to; + using interface10::internal::is_a; + using interface10::continue_node; + using interface10a::overwrite_node; + using interface10a::write_once_node; + using interface10::broadcast_node; + using interface10::buffer_node; + using interface10::queue_node; + using interface10::sequencer_node; + using interface10::priority_queue_node; + using interface11::limiter_node; + using namespace interface10::internal::graph_policy_namespace; + using interface10::join_node; + using interface10::input_port; + using interface10::copy_body; + using interface10::make_edge; + using interface10::remove_edge; + using interface10::internal::tag_value; #if __TBB_FLOW_GRAPH_CPP11_FEATURES - using interface9::composite_node; + using interface10::composite_node; #endif - using interface9::async_node; + using interface10::async_node; #if __TBB_PREVIEW_ASYNC_MSG - using interface9::async_msg; + using interface10::async_msg; #endif #if __TBB_PREVIEW_STREAMING_NODE - using interface9::port_ref; - using interface9::streaming_node; + using interface10::port_ref; + using interface10::streaming_node; #endif // __TBB_PREVIEW_STREAMING_NODE +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + using internal::node_priority_t; + using internal::no_priority; +#endif + } // flow } // tbb diff --git a/inst/include/tbb_local/tbb/flow_graph_abstractions.h b/inst/include/tbb_local/tbb/flow_graph_abstractions.h index f6eb3fb3..e690f0d3 100644 --- a/inst/include/tbb_local/tbb/flow_graph_abstractions.h +++ b/inst/include/tbb_local/tbb/flow_graph_abstractions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_abstractions_H @@ -23,7 +19,7 @@ namespace tbb { namespace flow { -namespace interface9 { +namespace interface10 { //! Pure virtual template classes that define interfaces for async communication class graph_proxy { @@ -47,10 +43,10 @@ class receiver_gateway : public graph_proxy { virtual bool try_put(const input_type&) = 0; }; -} //interface9 +} //interfaceX -using interface9::graph_proxy; -using interface9::receiver_gateway; +using interface10::graph_proxy; +using interface10::receiver_gateway; } //flow } //tbb diff --git a/inst/include/tbb_local/tbb/flow_graph_opencl_node.h b/inst/include/tbb_local/tbb/flow_graph_opencl_node.h index 89f4da7b..2c99fb33 100644 --- a/inst/include/tbb_local/tbb/flow_graph_opencl_node.h +++ b/inst/include/tbb_local/tbb/flow_graph_opencl_node.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_opencl_node_H @@ -43,94 +39,64 @@ namespace tbb { namespace flow { -namespace interface9 { - -class opencl_foundation; -class opencl_device_list; - -template -class opencl_buffer_impl; - -template -class opencl_program; - -class default_opencl_factory; - -class opencl_graph : public graph { -public: - //! Constructs a graph with isolated task_group_context - opencl_graph() : my_opencl_foundation( NULL ) {} - //! Constructs a graph with an user context - explicit opencl_graph( task_group_context& context ) : graph( context ), my_opencl_foundation( NULL ) {} - //! Destroys a graph - ~opencl_graph(); - //! Available devices - const opencl_device_list& available_devices(); - default_opencl_factory& opencl_factory(); -protected: - opencl_foundation *my_opencl_foundation; - opencl_foundation &get_opencl_foundation(); - - template - friend class opencl_buffer; - template - friend class opencl_image2d; - template - friend class opencl_node; - template - friend class opencl_factory; -}; +namespace interface10 { template class opencl_factory; -template -class dependency_msg; +namespace opencl_info { +class default_opencl_factory; +} +template +class opencl_program; -inline void enforce_cl_retcode( cl_int err, std::string msg ) { - if ( err != CL_SUCCESS ) { +inline void enforce_cl_retcode(cl_int err, std::string msg) { + if (err != CL_SUCCESS) { std::cerr << msg << "; error code: " << err << std::endl; throw msg; } } template -T event_info( cl_event e, cl_event_info i ) { +T event_info(cl_event e, cl_event_info i) { T res; - enforce_cl_retcode( clGetEventInfo( e, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL event information" ); + enforce_cl_retcode(clGetEventInfo(e, i, sizeof(res), &res, NULL), "Failed to get OpenCL event information"); return res; } template -T device_info( cl_device_id d, cl_device_info i ) { +T device_info(cl_device_id d, cl_device_info i) { T res; - enforce_cl_retcode( clGetDeviceInfo( d, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL device information" ); + enforce_cl_retcode(clGetDeviceInfo(d, i, sizeof(res), &res, NULL), "Failed to get OpenCL device information"); return res; } + template <> -std::string device_info( cl_device_id d, cl_device_info i ) { +inline std::string device_info(cl_device_id d, cl_device_info i) { size_t required; - enforce_cl_retcode( clGetDeviceInfo( d, i, 0, NULL, &required ), "Failed to get OpenCL device information" ); + enforce_cl_retcode(clGetDeviceInfo(d, i, 0, NULL, &required), "Failed to get OpenCL device information"); - char *buff = (char*)alloca( required ); - enforce_cl_retcode( clGetDeviceInfo( d, i, required, buff, NULL ), "Failed to get OpenCL device information" ); + char *buff = (char*)alloca(required); + enforce_cl_retcode(clGetDeviceInfo(d, i, required, buff, NULL), "Failed to get OpenCL device information"); return buff; } + template -T platform_info( cl_platform_id p, cl_platform_info i ) { +T platform_info(cl_platform_id p, cl_platform_info i) { T res; - enforce_cl_retcode( clGetPlatformInfo( p, i, sizeof( res ), &res, NULL ), "Failed to get OpenCL platform information" ); + enforce_cl_retcode(clGetPlatformInfo(p, i, sizeof(res), &res, NULL), "Failed to get OpenCL platform information"); return res; } + template <> -std::string platform_info( cl_platform_id p, cl_platform_info i ) { +inline std::string platform_info(cl_platform_id p, cl_platform_info i) { size_t required; - enforce_cl_retcode( clGetPlatformInfo( p, i, 0, NULL, &required ), "Failed to get OpenCL platform information" ); + enforce_cl_retcode(clGetPlatformInfo(p, i, 0, NULL, &required), "Failed to get OpenCL platform information"); - char *buff = (char*)alloca( required ); - enforce_cl_retcode( clGetPlatformInfo( p, i, required, buff, NULL ), "Failed to get OpenCL platform information" ); + char *buff = (char*)alloca(required); + enforce_cl_retcode(clGetPlatformInfo(p, i, required, buff, NULL), "Failed to get OpenCL platform information"); return buff; } @@ -144,24 +110,26 @@ class opencl_device { host = device_id_type( -1 ) }; - opencl_device() : my_device_id( unknown ) {} + opencl_device() : my_device_id( unknown ), my_cl_device_id( NULL ), my_cl_command_queue( NULL ) {} + + opencl_device( cl_device_id d_id ) : my_device_id( unknown ), my_cl_device_id( d_id ), my_cl_command_queue( NULL ) {} - opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ) {} + opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ), my_cl_command_queue( NULL ) {} std::string platform_profile() const { - return platform_info( platform(), CL_PLATFORM_PROFILE ); + return platform_info( platform_id(), CL_PLATFORM_PROFILE ); } std::string platform_version() const { - return platform_info( platform(), CL_PLATFORM_VERSION ); + return platform_info( platform_id(), CL_PLATFORM_VERSION ); } std::string platform_name() const { - return platform_info( platform(), CL_PLATFORM_NAME ); + return platform_info( platform_id(), CL_PLATFORM_NAME ); } std::string platform_vendor() const { - return platform_info( platform(), CL_PLATFORM_VENDOR ); + return platform_info( platform_id(), CL_PLATFORM_VENDOR ); } std::string platform_extensions() const { - return platform_info( platform(), CL_PLATFORM_EXTENSIONS ); + return platform_info( platform_id(), CL_PLATFORM_EXTENSIONS ); } template @@ -257,13 +225,12 @@ class opencl_device { my_cl_command_queue = cmd_queue; } -private: - opencl_device( cl_device_id d_id ) : my_device_id( unknown ), my_cl_device_id( d_id ) {} - - cl_platform_id platform() const { + cl_platform_id platform_id() const { return device_info( my_cl_device_id, CL_DEVICE_PLATFORM ); } +private: + device_id_type my_device_id; cl_device_id my_cl_device_id; cl_command_queue my_cl_command_queue; @@ -276,7 +243,6 @@ class opencl_device { friend class opencl_memory; template friend class opencl_program; - friend class opencl_foundation; #if TBB_USE_ASSERT template @@ -303,10 +269,66 @@ class opencl_device_list { const_iterator end() const { return my_container.end(); } const_iterator cbegin() const { return my_container.cbegin(); } const_iterator cend() const { return my_container.cend(); } + private: container_type my_container; }; +namespace internal { + +// Retrieve all OpenCL devices from machine +inline opencl_device_list find_available_devices() { + opencl_device_list opencl_devices; + + cl_uint num_platforms; + enforce_cl_retcode(clGetPlatformIDs(0, NULL, &num_platforms), "clGetPlatformIDs failed"); + + std::vector platforms(num_platforms); + enforce_cl_retcode(clGetPlatformIDs(num_platforms, platforms.data(), NULL), "clGetPlatformIDs failed"); + + cl_uint num_devices; + std::vector::iterator platforms_it = platforms.begin(); + cl_uint num_all_devices = 0; + while (platforms_it != platforms.end()) { + cl_int err = clGetDeviceIDs(*platforms_it, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); + if (err == CL_DEVICE_NOT_FOUND) { + platforms_it = platforms.erase(platforms_it); + } + else { + enforce_cl_retcode(err, "clGetDeviceIDs failed"); + num_all_devices += num_devices; + ++platforms_it; + } + } + + std::vector devices(num_all_devices); + std::vector::iterator devices_it = devices.begin(); + for (auto p = platforms.begin(); p != platforms.end(); ++p) { + enforce_cl_retcode(clGetDeviceIDs((*p), CL_DEVICE_TYPE_ALL, (cl_uint)std::distance(devices_it, devices.end()), &*devices_it, &num_devices), "clGetDeviceIDs failed"); + devices_it += num_devices; + } + + for (auto d = devices.begin(); d != devices.end(); ++d) { + opencl_devices.add(opencl_device((*d))); + } + + return opencl_devices; +} + +} // namespace internal + +// TODO: consider this namespace as public API +namespace opencl_info { + + inline const opencl_device_list& available_devices() { + // Static storage for all available OpenCL devices on machine + static const opencl_device_list my_devices = internal::find_available_devices(); + return my_devices; + } + +} // namespace opencl_info + + class callback_base : tbb::internal::no_copy { public: virtual void call() = 0; @@ -315,41 +337,30 @@ class callback_base : tbb::internal::no_copy { template class callback : public callback_base { - graph &my_graph; Callback my_callback; T my_data; public: - callback( graph &g, Callback c, const T& t ) : my_graph( g ), my_callback( c ), my_data( t ) { - // Extend the graph lifetime until the callback completion. - my_graph.increment_wait_count(); - } - ~callback() { - // Release the reference to the graph. - my_graph.decrement_wait_count(); - } + callback( Callback c, const T& t ) : my_callback( c ), my_data( t ) {} + void call() __TBB_override { my_callback( my_data ); } }; -template -class dependency_msg : public async_msg { +template +class opencl_async_msg : public async_msg { public: typedef T value_type; - dependency_msg() : my_callback_flag_ptr( std::make_shared< tbb::atomic>() ) { + opencl_async_msg() : my_callback_flag_ptr( std::make_shared< tbb::atomic>() ) { my_callback_flag_ptr->store(false); } - explicit dependency_msg( const T& data ) : my_data(data), my_callback_flag_ptr( std::make_shared>() ) { + explicit opencl_async_msg( const T& data ) : my_data(data), my_callback_flag_ptr( std::make_shared>() ) { my_callback_flag_ptr->store(false); } - dependency_msg( opencl_graph &g, const T& data ) : my_data(data), my_graph(&g), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - } - - dependency_msg( const T& data, cl_event event ) : my_data(data), my_event(event), my_is_event(true), my_callback_flag_ptr( std::make_shared>() ) { + opencl_async_msg( const T& data, cl_event event ) : my_data(data), my_event(event), my_is_event(true), my_callback_flag_ptr( std::make_shared>() ) { my_callback_flag_ptr->store(false); enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); } @@ -372,22 +383,22 @@ class dependency_msg : public async_msg { return my_data; } - dependency_msg( const dependency_msg &dmsg ) : async_msg(dmsg), - my_data(dmsg.my_data), my_event(dmsg.my_event), my_is_event( dmsg.my_is_event ), my_graph( dmsg.my_graph ), + opencl_async_msg( const opencl_async_msg &dmsg ) : async_msg(dmsg), + my_data(dmsg.my_data), my_event(dmsg.my_event), my_is_event( dmsg.my_is_event ), my_callback_flag_ptr(dmsg.my_callback_flag_ptr) { if ( my_is_event ) enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); } - dependency_msg( dependency_msg &&dmsg ) : async_msg(std::move(dmsg)), - my_data(std::move(dmsg.my_data)), my_event(dmsg.my_event), my_is_event(dmsg.my_is_event), my_graph(dmsg.my_graph), + opencl_async_msg( opencl_async_msg &&dmsg ) : async_msg(std::move(dmsg)), + my_data(std::move(dmsg.my_data)), my_event(dmsg.my_event), my_is_event(dmsg.my_is_event), my_callback_flag_ptr( std::move(dmsg.my_callback_flag_ptr) ) { dmsg.my_is_event = false; } - dependency_msg& operator=(const dependency_msg &dmsg) { + opencl_async_msg& operator=(const opencl_async_msg &dmsg) { async_msg::operator =(dmsg); // Release original event @@ -397,7 +408,6 @@ class dependency_msg : public async_msg { my_data = dmsg.my_data; my_event = dmsg.my_event; my_is_event = dmsg.my_is_event; - my_graph = dmsg.my_graph; // Retain copied event if ( my_is_event ) @@ -407,7 +417,7 @@ class dependency_msg : public async_msg { return *this; } - ~dependency_msg() { + ~opencl_async_msg() { if ( my_is_event ) enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); } @@ -425,10 +435,6 @@ class dependency_msg : public async_msg { clRetainEvent( my_event ); } - void set_graph( graph &g ) { - my_graph = &g; - } - void clear_event() const { if ( my_is_event ) { enforce_cl_retcode( clFlush( event_info( my_event, CL_EVENT_COMMAND_QUEUE ) ), "Failed to flush an OpenCL command queue" ); @@ -440,20 +446,19 @@ class dependency_msg : public async_msg { template void register_callback( Callback c ) const { __TBB_ASSERT( my_is_event, "The OpenCL event is not set" ); - __TBB_ASSERT( my_graph, "The graph is not set" ); - enforce_cl_retcode( clSetEventCallback( my_event, CL_COMPLETE, register_callback_func, new callback( *my_graph, c, my_data ) ), "Failed to set an OpenCL callback" ); + enforce_cl_retcode( clSetEventCallback( my_event, CL_COMPLETE, register_callback_func, new callback( c, my_data ) ), "Failed to set an OpenCL callback" ); } operator T&() { return data(); } operator const T&() const { return data(); } protected: - // Overridden in this derived class to inform that + // Overridden in this derived class to inform that // async calculation chain is over void finalize() const __TBB_override { receive_if_memory_object(*this); if (! my_callback_flag_ptr->fetch_and_store(true)) { - dependency_msg a(*this); + opencl_async_msg a(*this); if (my_is_event) { register_callback([a](const T& t) mutable { a.set(t); @@ -479,13 +484,12 @@ class dependency_msg : public async_msg { T my_data; mutable cl_event my_event; mutable bool my_is_event = false; - graph *my_graph = NULL; std::shared_ptr< tbb::atomic > my_callback_flag_ptr; }; template -K key_from_message( const dependency_msg &dmsg ) { +K key_from_message( const opencl_async_msg &dmsg ) { using tbb::flow::key_from_message; const T &t = dmsg.data( false ); __TBB_STATIC_ASSERT( true, "" ); @@ -511,7 +515,7 @@ class opencl_memory { void* get_host_ptr() { if ( !my_host_ptr ) { - dependency_msg d = receive( NULL ); + opencl_async_msg d = receive( NULL ); d.data(); __TBB_ASSERT( d.data() == my_host_ptr, NULL ); } @@ -520,9 +524,55 @@ class opencl_memory { Factory *factory() const { return my_factory; } - dependency_msg send( opencl_device d, const cl_event *e ); - dependency_msg receive( const cl_event *e ); - virtual void map_memory( opencl_device, dependency_msg & ) = 0; + opencl_async_msg receive(const cl_event *e) { + opencl_async_msg d; + if (e) { + d = opencl_async_msg(my_host_ptr, *e); + } else { + d = opencl_async_msg(my_host_ptr); + } + + // Concurrent receives are prohibited so we do not worry about synchronization. + if (my_curr_device_id.load() != opencl_device::host) { + map_memory(*my_factory->devices().begin(), d); + my_curr_device_id.store(opencl_device::host); + my_host_ptr = d.data(false); + } + // Release the sending event + if (my_sending_event_present) { + enforce_cl_retcode(clReleaseEvent(my_sending_event), "Failed to release an event"); + my_sending_event_present = false; + } + return d; + } + + opencl_async_msg send(opencl_device device, const cl_event *e) { + opencl_device::device_id_type device_id = device.my_device_id; + if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { + { + tbb::spin_mutex::scoped_lock lock(my_sending_lock); + if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { + __TBB_ASSERT(my_host_ptr, "The buffer has not been mapped"); + opencl_async_msg d(my_host_ptr); + my_factory->enqueue_unmap_buffer(device, *this, d); + my_sending_event = *d.get_event(); + my_sending_event_present = true; + enforce_cl_retcode(clRetainEvent(my_sending_event), "Failed to retain an event"); + my_host_ptr = NULL; + my_curr_device_id.store(device_id); + } + } + __TBB_ASSERT(my_sending_event_present, NULL); + } + + // !e means that buffer has come from the host + if (!e && my_sending_event_present) e = &my_sending_event; + + __TBB_ASSERT(!my_host_ptr, "The buffer has not been unmapped"); + return e ? opencl_async_msg(NULL, *e) : opencl_async_msg(NULL); + } + + virtual void map_memory( opencl_device, opencl_async_msg & ) = 0; protected: cl_mem my_cl_mem; tbb::atomic my_curr_device_id; @@ -556,8 +606,8 @@ class opencl_buffer_impl : public opencl_memory { return my_size; } - void map_memory( opencl_device device, dependency_msg &dmsg ) __TBB_override { - this->my_factory->enque_map_buffer( device, *this, dmsg ); + void map_memory( opencl_device device, opencl_async_msg &dmsg ) __TBB_override { + this->my_factory->enqueue_map_buffer( device, *this, dmsg ); } #if TBB_USE_ASSERT @@ -572,10 +622,10 @@ enum access_type { read_only }; -template +template class opencl_subbuffer; -template +template class opencl_buffer { public: typedef cl_mem native_object_type; @@ -604,7 +654,7 @@ class opencl_buffer { T& operator[] ( ptrdiff_t k ) { return begin()[k]; } opencl_buffer() {} - opencl_buffer( opencl_graph &g, size_t size ); + opencl_buffer( size_t size ); opencl_buffer( Factory &f, size_t size ) : my_impl( std::make_shared( size*sizeof(T), f ) ) {} cl_mem native_object() const { @@ -615,16 +665,16 @@ class opencl_buffer { return *this; } - void send( opencl_device device, dependency_msg &dependency ) const { + void send( opencl_device device, opencl_async_msg &dependency ) const { __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - dependency_msg d = my_impl->send( device, dependency.get_event() ); + opencl_async_msg d = my_impl->send( device, dependency.get_event() ); const cl_event *e = d.get_event(); if ( e ) dependency.set_event( *e ); else dependency.clear_event(); } - void receive( const dependency_msg &dependency ) const { + void receive( const opencl_async_msg &dependency ) const { __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - dependency_msg d = my_impl->receive( dependency.get_event() ); + opencl_async_msg d = my_impl->receive( dependency.get_event() ); const cl_event *e = d.get_event(); if ( e ) dependency.set_event( *e ); else dependency.clear_event(); @@ -690,11 +740,11 @@ typename std::enable_if::value, T>::type get_native_ob // send_if_memory_object checks if the T type has memory_object_type and call the send method for the object. template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, dependency_msg &dmsg ) { +typename std::enable_if::value>::type send_if_memory_object( opencl_device device, opencl_async_msg &dmsg ) { const T &t = dmsg.data( false ); typedef typename T::memory_object_type mem_obj_t; mem_obj_t mem_obj = t.memory_object(); - dependency_msg d( mem_obj ); + opencl_async_msg d( mem_obj ); if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); mem_obj.send( device, d ); if ( d.get_event() ) dmsg.set_event( *d.get_event() ); @@ -704,7 +754,7 @@ template typename std::enable_if::value>::type send_if_memory_object( opencl_device device, T &t ) { typedef typename T::memory_object_type mem_obj_t; mem_obj_t mem_obj = t.memory_object(); - dependency_msg dmsg( mem_obj ); + opencl_async_msg dmsg( mem_obj ); mem_obj.send( device, dmsg ); } @@ -713,11 +763,11 @@ typename std::enable_if::value>::type send_if_memory_o // receive_if_memory_object checks if the T type has memory_object_type and call the receive method for the object. template -typename std::enable_if::value>::type receive_if_memory_object( const dependency_msg &dmsg ) { +typename std::enable_if::value>::type receive_if_memory_object( const opencl_async_msg &dmsg ) { const T &t = dmsg.data( false ); typedef typename T::memory_object_type mem_obj_t; mem_obj_t mem_obj = t.memory_object(); - dependency_msg d( mem_obj ); + opencl_async_msg d( mem_obj ); if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); mem_obj.receive( d ); if ( d.get_event() ) dmsg.set_event( *d.get_event() ); @@ -756,7 +806,7 @@ class opencl_range { template class opencl_factory { public: - template using async_msg_type = dependency_msg>; + template using async_msg_type = opencl_async_msg>; typedef opencl_device device_type; class kernel : tbb::internal::no_assign { @@ -805,7 +855,7 @@ class opencl_factory { // it affects expectations for enqueue_kernel(.....) interface method typedef opencl_range range_type; - opencl_factory( opencl_graph &g ) : my_graph( g ) {} + opencl_factory() {} ~opencl_factory() { if ( my_devices.size() ) { for ( auto d = my_devices.begin(); d != my_devices.end(); ++d ) { @@ -827,7 +877,7 @@ class opencl_factory { private: template - void enque_map_buffer( opencl_device device, opencl_buffer_impl &buffer, dependency_msg& dmsg ) { + void enqueue_map_buffer( opencl_device device, opencl_buffer_impl &buffer, opencl_async_msg& dmsg ) { cl_event const* e1 = dmsg.get_event(); cl_event e2; cl_int err; @@ -841,7 +891,7 @@ class opencl_factory { template - void enque_unmap_buffer( opencl_device device, opencl_memory &memory, dependency_msg& dmsg ) { + void enqueue_unmap_buffer( opencl_device device, opencl_memory &memory, opencl_async_msg& dmsg ) { cl_event const* e1 = dmsg.get_event(); cl_event e2; enforce_cl_retcode( @@ -859,7 +909,7 @@ class opencl_factory { } template - void process_one_arg( const kernel_type& kernel, std::array& events, int& num_events, int& place, const dependency_msg& msg ) { + void process_one_arg( const kernel_type& kernel, std::array& events, int& num_events, int& place, const opencl_async_msg& msg ) { __TBB_ASSERT((static_cast::size_type>(num_events) < events.size()), NULL); const cl_event * const e = msg.get_event(); @@ -883,9 +933,8 @@ class opencl_factory { void update_one_arg( cl_event, T& ) {} template - void update_one_arg( cl_event e, dependency_msg& msg ) { + void update_one_arg( cl_event e, opencl_async_msg& msg ) { msg.set_event( e ); - msg.set_graph( my_graph ); } template @@ -945,7 +994,7 @@ class opencl_factory { } template - bool get_event_from_one_arg( cl_event& e, const dependency_msg& msg) { + bool get_event_from_one_arg( cl_event& e, const opencl_async_msg& msg) { cl_event const *e_ptr = msg.get_event(); if ( e_ptr != NULL ) { @@ -1028,12 +1077,79 @@ class opencl_factory { return my_cl_context; } - void init_once(); + void init_once() { + { + tbb::spin_mutex::scoped_lock lock(my_devices_mutex); + if (!my_devices.size()) + my_devices = DeviceFilter()( opencl_info::available_devices() ); + } + + enforce_cl_retcode(my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list"); + cl_platform_id platform_id = my_devices.begin()->platform_id(); + for (opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it) + enforce_cl_retcode(it->platform_id() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform"); + + std::vector cl_device_ids; + for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { + cl_device_ids.push_back((*d).my_cl_device_id); + } + + cl_context_properties context_properties[3] = { CL_CONTEXT_PLATFORM, (cl_context_properties)platform_id, (cl_context_properties)NULL }; + cl_int err; + cl_context ctx = clCreateContext(context_properties, + (cl_uint)cl_device_ids.size(), + cl_device_ids.data(), + NULL, NULL, &err); + enforce_cl_retcode(err, "Failed to create context"); + my_cl_context = ctx; + + size_t device_counter = 0; + for (auto d = my_devices.begin(); d != my_devices.end(); d++) { + (*d).my_device_id = device_counter++; + cl_int err2; + cl_command_queue cq; +#if CL_VERSION_2_0 + if ((*d).major_version() >= 2) { + if ((*d).out_of_order_exec_mode_on_host_present()) { + cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, 0 }; + cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); + } else { + cl_queue_properties props[] = { 0 }; + cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); + } + } else +#endif + { + cl_command_queue_properties props = (*d).out_of_order_exec_mode_on_host_present() ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0; + // Suppress "declared deprecated" warning for the next line. +#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +#if _MSC_VER || __INTEL_COMPILER +#pragma warning( push ) +#if __INTEL_COMPILER +#pragma warning (disable: 1478) +#else +#pragma warning (disable: 4996) +#endif +#endif + cq = clCreateCommandQueue(ctx, (*d).my_cl_device_id, props, &err2); +#if _MSC_VER || __INTEL_COMPILER +#pragma warning( pop ) +#endif +#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT +#pragma GCC diagnostic pop +#endif + } + enforce_cl_retcode(err2, "Failed to create command queue"); + (*d).my_cl_command_queue = cq; + } + } std::once_flag my_once_flag; opencl_device_list my_devices; cl_context my_cl_context; - opencl_graph &my_graph; tbb::spin_mutex my_devices_mutex; @@ -1043,238 +1159,62 @@ class opencl_factory { friend class opencl_buffer_impl; template friend class opencl_memory; -}; +}; // class opencl_factory -template -dependency_msg opencl_memory::receive( const cl_event *e ) { - dependency_msg d = e ? dependency_msg( my_host_ptr, *e ) : dependency_msg( my_host_ptr ); - // Concurrent receives are prohibited so we do not worry about synchronization. - if ( my_curr_device_id.load() != opencl_device::host ) { - map_memory( *my_factory->devices().begin(), d ); - my_curr_device_id.store( opencl_device::host ); - my_host_ptr = d.data( false ); - } - // Release the sending event - if ( my_sending_event_present ) { - enforce_cl_retcode( clReleaseEvent( my_sending_event ), "Failed to release an event" ); - my_sending_event_present = false; - } - return d; -} +// TODO: consider this namespace as public API +namespace opencl_info { + +// Default types template -dependency_msg opencl_memory::send( opencl_device device, const cl_event *e ) { - opencl_device::device_id_type device_id = device.my_device_id; - if ( !my_factory->is_same_context( my_curr_device_id.load(), device_id ) ) { - { - tbb::spin_mutex::scoped_lock lock( my_sending_lock ); - if ( !my_factory->is_same_context( my_curr_device_id.load(), device_id ) ) { - __TBB_ASSERT( my_host_ptr, "The buffer has not been mapped" ); - dependency_msg d( my_host_ptr ); - my_factory->enque_unmap_buffer( device, *this, d ); - my_sending_event = *d.get_event(); - my_sending_event_present = true; - enforce_cl_retcode( clRetainEvent( my_sending_event ), "Failed to retain an event" ); - my_host_ptr = NULL; - my_curr_device_id.store(device_id); - } - } - __TBB_ASSERT( my_sending_event_present, NULL ); +struct default_device_selector { + opencl_device operator()(Factory& f) { + __TBB_ASSERT(!f.devices().empty(), "No available devices"); + return *(f.devices().begin()); } +}; - // !e means that buffer has come from the host - if ( !e && my_sending_event_present ) e = &my_sending_event; - - __TBB_ASSERT( !my_host_ptr, "The buffer has not been unmapped" ); - return e ? dependency_msg( NULL, *e ) : dependency_msg( NULL ); -} - -struct default_opencl_factory_device_filter { - opencl_device_list operator()( const opencl_device_list &devices ) { +struct default_device_filter { + opencl_device_list operator()(const opencl_device_list &devices) { opencl_device_list dl; - dl.add( *devices.begin() ); + cl_platform_id platform_id = devices.begin()->platform_id(); + for (opencl_device_list::const_iterator it = devices.cbegin(); it != devices.cend(); ++it) { + if (it->platform_id() == platform_id) { + dl.add(*it); + } + } return dl; } }; -class default_opencl_factory : public opencl_factory < default_opencl_factory_device_filter > { -public: - template using async_msg_type = dependency_msg; - - default_opencl_factory( opencl_graph &g ) : opencl_factory( g ) {} -private: - default_opencl_factory( const default_opencl_factory& ); - default_opencl_factory& operator=(const default_opencl_factory&); -}; - -class opencl_foundation : tbb::internal::no_assign { - struct default_device_selector_type { - opencl_device operator()( default_opencl_factory& f ) { - __TBB_ASSERT( ! f.devices().empty(), "No available devices" ); - return *( f.devices().begin() ); - } - }; +class default_opencl_factory : public opencl_factory < default_device_filter >, tbb::internal::no_copy { public: - opencl_foundation(opencl_graph &g) : my_default_opencl_factory(g), my_default_device_selector() { - cl_uint num_platforms; - enforce_cl_retcode(clGetPlatformIDs(0, NULL, &num_platforms), "clGetPlatformIDs failed"); - - std::vector platforms(num_platforms); - enforce_cl_retcode(clGetPlatformIDs(num_platforms, platforms.data(), NULL), "clGetPlatformIDs failed"); - - cl_uint num_devices; - std::vector::iterator platforms_it = platforms.begin(); - cl_uint num_all_devices = 0; - while (platforms_it != platforms.end()) { - cl_int err = clGetDeviceIDs(*platforms_it, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); - if (err == CL_DEVICE_NOT_FOUND) { - platforms_it = platforms.erase(platforms_it); - } else { - enforce_cl_retcode(err, "clGetDeviceIDs failed"); - num_all_devices += num_devices; - ++platforms_it; - } - } - - std::vector devices(num_all_devices); - std::vector::iterator devices_it = devices.begin(); - for (auto p = platforms.begin(); p != platforms.end(); ++p) { - enforce_cl_retcode(clGetDeviceIDs((*p), CL_DEVICE_TYPE_ALL, (cl_uint)std::distance(devices_it, devices.end()), &*devices_it, &num_devices), "clGetDeviceIDs failed"); - devices_it += num_devices; - } + template using async_msg_type = opencl_async_msg; - for (auto d = devices.begin(); d != devices.end(); ++d) { - my_devices.add(opencl_device((*d))); - } - } - - default_opencl_factory &get_default_opencl_factory() { - return my_default_opencl_factory; - } - - const opencl_device_list &get_all_devices() { - return my_devices; - } - - default_device_selector_type get_default_device_selector() { return my_default_device_selector; } + friend default_opencl_factory& default_factory(); private: - default_opencl_factory my_default_opencl_factory; - opencl_device_list my_devices; - - const default_device_selector_type my_default_device_selector; + default_opencl_factory() = default; }; -opencl_foundation &opencl_graph::get_opencl_foundation() { - opencl_foundation* INITIALIZATION = (opencl_foundation*)1; - if ( my_opencl_foundation <= INITIALIZATION ) { - if ( tbb::internal::as_atomic( my_opencl_foundation ).compare_and_swap( INITIALIZATION, NULL ) == 0 ) { - my_opencl_foundation = new opencl_foundation( *this ); - } - else { - tbb::internal::spin_wait_while_eq( my_opencl_foundation, INITIALIZATION ); - } - } - - __TBB_ASSERT( my_opencl_foundation > INITIALIZATION, "opencl_foundation is not initialized"); - return *my_opencl_foundation; +inline default_opencl_factory& default_factory() { + static default_opencl_factory default_factory; + return default_factory; } -opencl_graph::~opencl_graph() { - if ( my_opencl_foundation ) - delete my_opencl_foundation; -} - -template -void opencl_factory::init_once() { - { - tbb::spin_mutex::scoped_lock lock( my_devices_mutex ); - if ( !my_devices.size() ) - my_devices = DeviceFilter()(my_graph.get_opencl_foundation().get_all_devices()); - } - - enforce_cl_retcode( my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list" ); - cl_platform_id platform_id = my_devices.begin()->platform(); - for ( opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it ) - enforce_cl_retcode( it->platform() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform" ); - - std::vector cl_device_ids; - for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { - cl_device_ids.push_back((*d).my_cl_device_id); - } - - cl_context_properties context_properties[3] = { CL_CONTEXT_PLATFORM, (cl_context_properties)platform_id, (cl_context_properties)NULL }; - cl_int err; - cl_context ctx = clCreateContext( context_properties, - (cl_uint)cl_device_ids.size(), - cl_device_ids.data(), - NULL, NULL, &err ); - enforce_cl_retcode( err, "Failed to create context" ); - my_cl_context = ctx; - - size_t device_counter = 0; - for ( auto d = my_devices.begin(); d != my_devices.end(); d++ ) { - (*d).my_device_id = device_counter++; - cl_int err2; - cl_command_queue cq; -#if CL_VERSION_2_0 - if ( (*d).major_version() >= 2 ) { - if ( (*d).out_of_order_exec_mode_on_host_present() ) { - cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, 0 }; - cq = clCreateCommandQueueWithProperties( ctx, (*d).my_cl_device_id, props, &err2 ); - } else { - cl_queue_properties props[] = { 0 }; - cq = clCreateCommandQueueWithProperties( ctx, (*d).my_cl_device_id, props, &err2 ); - } - } else -#endif - { - cl_command_queue_properties props = (*d).out_of_order_exec_mode_on_host_present() ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0; - // Suppress "declared deprecated" warning for the next line. -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( push ) -#if __INTEL_COMPILER -#pragma warning (disable: 1478) -#else -#pragma warning (disable: 4996) -#endif -#endif - cq = clCreateCommandQueue( ctx, (*d).my_cl_device_id, props, &err2 ); -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( pop ) -#endif -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic pop -#endif - } - enforce_cl_retcode( err2, "Failed to create command queue" ); - (*d).my_cl_command_queue = cq; - } -} - -const opencl_device_list &opencl_graph::available_devices() { - return get_opencl_foundation().get_all_devices(); -} - -default_opencl_factory &opencl_graph::opencl_factory() { - return get_opencl_foundation().get_default_opencl_factory(); -} +} // namespace opencl_info template -opencl_buffer::opencl_buffer( opencl_graph &g, size_t size ) : my_impl( std::make_shared( size*sizeof(T), g.get_opencl_foundation().get_default_opencl_factory() ) ) {} +opencl_buffer::opencl_buffer( size_t size ) : my_impl( std::make_shared( size*sizeof(T), opencl_info::default_factory() ) ) {} + - enum class opencl_program_type { SOURCE, PRECOMPILED, SPIR }; -template +template class opencl_program : tbb::internal::no_assign { public: typedef typename Factory::kernel_type kernel_type; @@ -1283,10 +1223,10 @@ class opencl_program : tbb::internal::no_assign { opencl_program( Factory& factory, const char* program_name ) : opencl_program( factory, std::string( program_name ) ) {} opencl_program( Factory& factory, const std::string& program_name ) : opencl_program( factory, opencl_program_type::SOURCE, program_name ) {} - opencl_program( opencl_graph& graph, opencl_program_type type, const std::string& program_name ) : opencl_program( graph.opencl_factory(), type, program_name ) {} - opencl_program( opencl_graph& graph, const char* program_name ) : opencl_program( graph.opencl_factory(), program_name ) {} - opencl_program( opencl_graph& graph, const std::string& program_name ) : opencl_program( graph.opencl_factory(), program_name ) {} - opencl_program( opencl_graph& graph, opencl_program_type type ) : opencl_program( graph.opencl_factory(), type ) {} + opencl_program( opencl_program_type type, const std::string& program_name ) : opencl_program( opencl_info::default_factory(), type, program_name ) {} + opencl_program( const char* program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} + opencl_program( const std::string& program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} + opencl_program( opencl_program_type type ) : opencl_program( opencl_info::default_factory(), type ) {} opencl_program( const opencl_program &src ) : my_factory( src.my_factory ), my_type( src.type ), my_arg_str( src.my_arg_str ), my_cl_program( src.my_cl_program ) { // Set my_do_once_flag to the called state. @@ -1467,68 +1407,73 @@ class opencl_node< tuple, JP, Factory > : public streaming_node< tuple public: typedef typename base_type::kernel_type kernel_type; - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) - {} + opencl_node( graph &g, const kernel_type& kernel ) + : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) + { + tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); + } - opencl_node( opencl_graph &g, const kernel_type& kernel, Factory &f ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), f ) - {} + opencl_node( graph &g, const kernel_type& kernel, Factory &f ) + : base_type( g, kernel, opencl_info::default_device_selector (), f ) + { + tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); + } template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d, Factory &f) + opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d, Factory &f) : base_type( g, kernel, d, f) - {} + { + tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); + } }; template -class opencl_node< tuple, JP > : public opencl_node < tuple, JP, default_opencl_factory > { - typedef opencl_node < tuple, JP, default_opencl_factory > base_type; +class opencl_node< tuple, JP > : public opencl_node < tuple, JP, opencl_info::default_opencl_factory > { + typedef opencl_node < tuple, JP, opencl_info::default_opencl_factory > base_type; public: typedef typename base_type::kernel_type kernel_type; - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) + opencl_node( graph &g, const kernel_type& kernel ) + : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) {} template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, g.get_opencl_foundation().get_default_opencl_factory() ) + opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) + : base_type( g, kernel, d, opencl_info::default_factory() ) {} }; template -class opencl_node< tuple > : public opencl_node < tuple, queueing, default_opencl_factory > { - typedef opencl_node < tuple, queueing, default_opencl_factory > base_type; +class opencl_node< tuple > : public opencl_node < tuple, queueing, opencl_info::default_opencl_factory > { + typedef opencl_node < tuple, queueing, opencl_info::default_opencl_factory > base_type; public: typedef typename base_type::kernel_type kernel_type; - opencl_node( opencl_graph &g, const kernel_type& kernel ) - : base_type( g, kernel, g.get_opencl_foundation().get_default_device_selector(), g.get_opencl_foundation().get_default_opencl_factory() ) + opencl_node( graph &g, const kernel_type& kernel ) + : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) {} template - opencl_node( opencl_graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, g.get_opencl_foundation().get_default_opencl_factory() ) + opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) + : base_type( g, kernel, d, opencl_info::default_factory() ) {} }; -} // namespace interface9 - -using interface9::opencl_graph; -using interface9::opencl_node; -using interface9::read_only; -using interface9::read_write; -using interface9::write_only; -using interface9::opencl_buffer; -using interface9::opencl_subbuffer; -using interface9::opencl_device; -using interface9::opencl_device_list; -using interface9::opencl_program; -using interface9::opencl_program_type; -using interface9::dependency_msg; -using interface9::opencl_factory; -using interface9::opencl_range; +} // namespace interface10 + +using interface10::opencl_node; +using interface10::read_only; +using interface10::read_write; +using interface10::write_only; +using interface10::opencl_buffer; +using interface10::opencl_subbuffer; +using interface10::opencl_device; +using interface10::opencl_device_list; +using interface10::opencl_program; +using interface10::opencl_program_type; +using interface10::opencl_async_msg; +using interface10::opencl_factory; +using interface10::opencl_range; } // namespace flow } // namespace tbb diff --git a/inst/include/tbb_local/tbb/global_control.h b/inst/include/tbb_local/tbb/global_control.h index fe742020..505720cb 100644 --- a/inst/include/tbb_local/tbb/global_control.h +++ b/inst/include/tbb_local/tbb/global_control.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,19 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_global_control_H #define __TBB_global_control_H -#if !TBB_PREVIEW_GLOBAL_CONTROL && !__TBB_BUILD -#error Set TBB_PREVIEW_GLOBAL_CONTROL before including global_control.h -#endif - #include "tbb_stddef.h" namespace tbb { @@ -41,8 +33,8 @@ class global_control { global_control(parameter p, size_t value) : my_value(value), my_next(NULL), my_param(p) { __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size if (p==thread_stack_size) return; #elif __TBB_x86_64 && (_WIN32 || _WIN64) @@ -56,8 +48,8 @@ class global_control { ~global_control() { __TBB_ASSERT(my_param < parameter_max, "Invalid parameter. Probably the object was corrupted."); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size if (my_param==thread_stack_size) return; #endif diff --git a/inst/include/tbb_local/tbb/index.html b/inst/include/tbb_local/tbb/index.html index 2eddd450..7bbb672d 100644 --- a/inst/include/tbb_local/tbb/index.html +++ b/inst/include/tbb_local/tbb/index.html @@ -19,7 +19,7 @@

Directories

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/inst/include/tbb_local/tbb/internal/_aggregator_impl.h b/inst/include/tbb_local/tbb/internal/_aggregator_impl.h index 40bbd491..cece90f3 100644 --- a/inst/include/tbb_local/tbb/internal/_aggregator_impl.h +++ b/inst/include/tbb_local/tbb/internal/_aggregator_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__aggregator_impl_H diff --git a/inst/include/tbb_local/tbb/internal/_allocator_traits.h b/inst/include/tbb_local/tbb/internal/_allocator_traits.h new file mode 100644 index 00000000..272077a6 --- /dev/null +++ b/inst/include/tbb_local/tbb/internal/_allocator_traits.h @@ -0,0 +1,156 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_allocator_traits_H +#define __TBB_allocator_traits_H + +#include "../tbb_stddef.h" // true/false_type + +#if __TBB_ALLOCATOR_TRAITS_PRESENT +#include // for allocator_traits +#endif + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include // for std::move +#endif + +// For allocator_swap helper +#include __TBB_STD_SWAP_HEADER + +namespace tbb { +namespace internal { + +//! Internal implementation of allocator traits, propagate_on_* use internal boolean_constant. +//! In order to avoid code duplication, check what implementation of boolean constant will likely be passed. +#if __TBB_ALLOCATOR_TRAITS_PRESENT +typedef std::true_type traits_true_type; +typedef std::false_type traits_false_type; +#else +typedef tbb::internal::true_type traits_true_type; +typedef tbb::internal::false_type traits_false_type; +#endif + +//! Copy assignment implementation for allocator if propagate_on_container_copy_assignment == true_type +//! Noop if pocca == false_type +template +inline void allocator_copy_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + my_allocator = other_allocator; +} +template +inline void allocator_copy_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO COPY */} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +//! Move assignment implementation for allocator if propagate_on_container_move_assignment == true_type. +//! Noop if pocma == false_type. +template +inline void allocator_move_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + my_allocator = std::move(other_allocator); +} +template +inline void allocator_move_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO MOVE */ } +#endif + +//! Swap implementation for allocators if propagate_on_container_swap == true_type. +//! Noop if pocs == false_type. +template +inline void allocator_swap(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + using std::swap; + swap(my_allocator, other_allocator); +} +template +inline void allocator_swap(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO SWAP */ } + +#if __TBB_ALLOCATOR_TRAITS_PRESENT +using std::allocator_traits; +#else +//! Internal allocator_traits implementation, which relies on C++03 standard +//! [20.1.5] allocator requirements +template +struct allocator_traits { + // C++03 allocator doesn't have to be assignable or swappable, therefore + // define these traits as false_type to do not require additional operations + // that are not supposed to be in. + typedef tbb::internal::false_type propagate_on_container_move_assignment; + typedef tbb::internal::false_type propagate_on_container_copy_assignment; + typedef tbb::internal::false_type propagate_on_container_swap; + + typedef Alloc allocator_type; + typedef typename allocator_type::value_type value_type; + + typedef typename allocator_type::pointer pointer; + typedef typename allocator_type::const_pointer const_pointer; + typedef typename allocator_type::difference_type difference_type; + typedef typename allocator_type::size_type size_type; + + template struct rebind_alloc { + typedef typename Alloc::template rebind::other other; + }; + + static pointer allocate(Alloc& a, size_type n) { + return a.allocate(n); + } + + static void deallocate(Alloc& a, pointer p, size_type n) { + a.deallocate(p, n); + } + + template + static void construct(Alloc&, PT* p) { + ::new (static_cast(p)) PT(); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1)); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, __TBB_FORWARDING_REF(T2) t2) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2)); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, + __TBB_FORWARDING_REF(T2) t2, __TBB_FORWARDING_REF(T3) t3) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2), + tbb::internal::forward(t3)); + } + + template + static void destroy(Alloc&, T* p) { + p->~T(); + tbb::internal::suppress_unused_warning(p); + } + + static Alloc select_on_container_copy_construction(const Alloc& a) { return a; } +}; +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + +//! C++03/C++11 compliant rebind helper, even if no std::allocator_traits available +//! or rebind is not defined for allocator type +template +struct allocator_rebind { +#if __TBB_ALLOCATOR_TRAITS_PRESENT + typedef typename allocator_traits::template rebind_alloc type; +#else + typedef typename allocator_traits::template rebind_alloc::other type; +#endif +}; + +}} // namespace tbb::internal + +#endif // __TBB_allocator_traits_H + diff --git a/inst/include/tbb_local/tbb/internal/_concurrent_queue_impl.h b/inst/include/tbb_local/tbb/internal/_concurrent_queue_impl.h index 4283f87c..c1aafc9e 100644 --- a/inst/include/tbb_local/tbb/internal/_concurrent_queue_impl.h +++ b/inst/include/tbb_local/tbb/internal/_concurrent_queue_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__concurrent_queue_impl_H @@ -34,19 +30,8 @@ #include "../tbb_profiling.h" #include #include __TBB_STD_SWAP_HEADER - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - namespace tbb { #if !__TBB_TEMPLATE_FRIENDS_BROKEN @@ -525,7 +510,7 @@ concurrent_queue_base_v3::concurrent_queue_base_v3() { __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset((void*) my_rep, 0, sizeof(concurrent_queue_rep)); + memset(static_cast(my_rep),0,sizeof(concurrent_queue_rep)); my_rep->item_size = item_size; my_rep->items_per_page = item_size<= 8 ? 32 : item_size<= 16 ? 16 : @@ -900,7 +885,7 @@ class concurrent_queue_base_v3: no_copy { //! Get size of queue ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - //! Check if the queue is emtpy + //! Check if the queue is empty bool __TBB_EXPORTED_METHOD internal_empty() const; //! Set the queue capacity diff --git a/inst/include/tbb_local/tbb/internal/_concurrent_skip_list_impl.h b/inst/include/tbb_local/tbb/internal/_concurrent_skip_list_impl.h new file mode 100644 index 00000000..685c45ae --- /dev/null +++ b/inst/include/tbb_local/tbb/internal/_concurrent_skip_list_impl.h @@ -0,0 +1,1043 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_skip_list_H +#define __TBB_concurrent_skip_list_H + +#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "../tbb_config.h" +#include "../tbb_stddef.h" +#include "../tbb_allocator.h" +#include "../spin_mutex.h" +#include "../tbb_exception.h" +#include "../enumerable_thread_specific.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_node_handle_impl.h" +#include // Need std::pair +#include +#include +#include // Need std::allocator_traits +#include +#include +#include +#include +#include +#include +#include +#include + +#if _MSC_VER +#pragma warning(disable: 4189) // warning 4189 -- local variable is initialized but not referenced +#pragma warning(disable: 4127) // warning 4127 -- while (true) has a constant expression in it +#endif + +namespace tbb { +namespace interface10 { +namespace internal { + +template +class skip_list_node { + +public: + using value_type = Value; + using size_type = std::size_t; + using reference = value_type & ; + using const_reference = const value_type & ; + using pointer = value_type * ; + using const_pointer = const value_type *; + using node_pointer = skip_list_node * ; + using atomic_node_pointer = std::atomic; + + using mutex_type = Mutex; + using lock_type = std::unique_lock; + + skip_list_node(size_type levels) : my_height(levels), my_fullyLinked(false) { + for (size_type lev = 0; lev < my_height; ++lev) + new(&my_next(lev)) atomic_node_pointer(nullptr); + __TBB_ASSERT(height() == levels, "Wrong node height"); + } + + ~skip_list_node() { + for(size_type lev = 0; lev < my_height; ++lev) + my_next(lev).~atomic(); + } + + skip_list_node(const skip_list_node&) = delete; + + skip_list_node(skip_list_node&&) = delete; + + skip_list_node& operator=(const skip_list_node&) = delete; + + pointer storage() { + return reinterpret_cast(&my_val); + } + + reference value() { + return *storage(); + } + + node_pointer next(size_type level) const { + __TBB_ASSERT(level < height(), "Cannot get next on the level greater than height"); + return my_next(level).load(std::memory_order_acquire); + } + + void set_next(size_type level, node_pointer next) { + __TBB_ASSERT(level < height(), "Cannot set next on the level greater than height"); + + my_next(level).store(next, std::memory_order_release); + } + + /** @return number of layers */ + size_type height() const { + return my_height; + } + + bool fully_linked() const { + return my_fullyLinked.load(std::memory_order_acquire); + } + + void mark_linked() { + my_fullyLinked.store(true, std::memory_order_release); + } + + lock_type acquire() { + return lock_type(my_mutex); + } + +private: + using aligned_storage_type = typename std::aligned_storage::type; + + atomic_node_pointer& my_next(size_type level) { + atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + const atomic_node_pointer& my_next(size_type level) const { + const atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + mutex_type my_mutex; + aligned_storage_type my_val; + size_type my_height; + std::atomic_bool my_fullyLinked; +}; + +template +class skip_list_iterator { + using node_type = NodeType; + using node_ptr = node_type*; +public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename node_type::value_type; + using difference_type = std::ptrdiff_t; + using pointer = typename std::conditional::type; + using reference = typename std::conditional::type; + + skip_list_iterator() : my_node_ptr(nullptr) {} + + // TODO: the code above does not compile in VS2015 (seems like a bug) - consider enabling it for all other platforms + // template ::type> + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + template ::type> + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + reference operator*() const { return *(my_node_ptr->storage()); } + pointer operator->() const { return &**this; } + + skip_list_iterator& operator++() { + __TBB_ASSERT(my_node_ptr != nullptr, NULL); + my_node_ptr = my_node_ptr->next(0); + return *this; + } + + skip_list_iterator operator++(int) { + skip_list_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + skip_list_iterator(node_type* n) : my_node_ptr(n) {} + + node_ptr my_node_ptr; + + template + friend class concurrent_skip_list; + + friend class skip_list_iterator; + + friend class const_range; + friend class range; + + template + friend bool operator==(const skip_list_iterator&, const skip_list_iterator&); + + template + friend bool operator!=(const skip_list_iterator&, const skip_list_iterator&); +}; + +template +bool operator==(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr == rhs.my_node_ptr; +} + +template +bool operator!=(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr != rhs.my_node_ptr; +} + +template +class concurrent_skip_list { +protected: + using traits_type = Traits; + using allocator_type = typename traits_type::allocator_type; + using allocator_traits_type = std::allocator_traits; + using key_compare = typename traits_type::compare_type; + using value_compare = typename traits_type::value_compare; + using key_type = typename traits_type::key_type; + using value_type = typename traits_type::value_type; + using node_type = typename traits_type::node_type; + using list_node_type = skip_list_node; + + using iterator = skip_list_iterator; + using const_iterator = skip_list_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using random_level_generator_type = typename traits_type::random_level_generator_type; + using node_allocator_type = typename std::allocator_traits::template rebind_alloc; + using node_allocator_traits = typename std::allocator_traits::template rebind_traits; + using node_ptr = list_node_type*; + + static constexpr size_type MAX_LEVEL = traits_type::MAX_LEVEL; + + using array_type = std::array; + using lock_array = std::array; + +public: + static bool const allow_multimapping = traits_type::allow_multimapping; + + /** + * Default constructor. Construct empty skip list. + */ + concurrent_skip_list() : my_size(0) { + create_dummy_head(); + } + + explicit concurrent_skip_list(const key_compare& comp, const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + } + + template + concurrent_skip_list(InputIt first, InputIt last, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + internal_copy(first, last); + } + + /** Copy constructor */ + concurrent_skip_list(const concurrent_skip_list& other) + : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())), + my_compare(other.my_compare), my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(const concurrent_skip_list& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(concurrent_skip_list&& other) + : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + internal_move(std::move(other)); + } + + concurrent_skip_list(concurrent_skip_list&& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + if (alloc == other.get_allocator()) { + internal_move(std::move(other)); + } else { + my_size = 0; + create_dummy_head(); + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + ~concurrent_skip_list() { + clear(); + delete_dummy_head(); + } + + concurrent_skip_list& operator=(const concurrent_skip_list& other) { + if (this != &other) { + using pocca_type = typename node_allocator_traits::propagate_on_container_copy_assignment; + clear(); + tbb::internal::allocator_copy_assignment(my_node_allocator, other.my_node_allocator, pocca_type()); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_copy(other); + } + return *this; + } + + concurrent_skip_list& operator=(concurrent_skip_list&& other) { + if (this != &other) { + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + clear(); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_move_assign(std::move(other), pocma_type()); + } + return *this; + } + + concurrent_skip_list& operator=(std::initializer_list il) + { + clear(); + insert(il.begin(),il.end()); + return *this; + } + + std::pair insert(const value_type& value) { + return internal_insert(value); + } + + std::pair insert(value_type&& value) { + return internal_insert(std::move(value)); + } + + iterator insert(const_iterator, const_reference value) { + // Ignore hint + return insert(value).first; + } + + iterator insert(const_iterator, value_type&& value) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert(InputIterator first, InputIterator last) { + for (InputIterator it = first; it != last; ++it) + insert(*it); + } + + void insert(std::initializer_list init) { + insert(init.begin(), init.end()); + } + + std::pair insert(node_type&& nh) { + if(!nh.empty()) { + std::pair insert_result = internal_insert_node(nh.my_node); + if(insert_result.second) { + nh.deactivate(); + } + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert(const_iterator, node_type&& nh) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace(Args&&... args) { + return internal_insert(std::forward(args)...); + } + + template + iterator emplace_hint(const_iterator, Args&&... args) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase(iterator pos) { + std::pair extract_result = internal_extract(pos); + if(extract_result.first) { // node was extracted + delete_node(extract_result.first); + return extract_result.second; + } + return end(); + } + + iterator unsafe_erase(const_iterator first, const_iterator last) { + while(first != last) { + first = unsafe_erase(get_iterator(first)); + } + return get_iterator(first); + } + + size_type unsafe_erase(const key_type& key) { + std::pair range = equal_range(key); + size_type sz = std::distance(range.first, range.second); + unsafe_erase(range.first, range.second); + return sz; + } + + node_type unsafe_extract(const_iterator pos) { + std::pair extract_result = internal_extract(pos); + return extract_result.first ? node_type(extract_result.first) : node_type(); + } + + node_type unsafe_extract(const key_type& key) { + return unsafe_extract(find(key)); + } + + iterator lower_bound(const key_type& key) { + return internal_get_bound(key, my_compare); + } + + const_iterator lower_bound(const key_type& key) const { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + iterator lower_bound(const K& key) { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + const_iterator lower_bound(const K& key) const { + return internal_get_bound(key, my_compare); + } + + iterator upper_bound(const key_type& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + const_iterator upper_bound(const key_type& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + iterator upper_bound(const K& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + const_iterator upper_bound(const K& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + iterator find(const key_type& key) { + return internal_find(key); + } + + const_iterator find(const key_type& key) const { + return internal_find(key); + } + + template::value, K>::type> + iterator find(const K& key) { + return internal_find(key); + } + + template::value, K>::type> + const_iterator find(const K& key) const { + return internal_find(key); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template::value, K>::type> + size_type count(const K& key) const { + return internal_count(key); + } + + bool contains(const key_type& key) const { + return find(key) != end(); + } + + template::value, K>::type> + bool contains(const K& key) const { + return find(key) != end(); + } + + void clear() noexcept { + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + node_ptr current = dummy_head->next(0); + while (current) { + __TBB_ASSERT(current->height() > 0, NULL); + node_ptr next = current->next(0); + delete_node(current); + current = next; + } + + my_size = 0; + for (size_type i = 0; i < dummy_head->height(); ++i) { + dummy_head->set_next(i, nullptr); + } + } + + iterator begin() { + return iterator(dummy_head->next(0)); + } + + const_iterator begin() const { + return const_iterator(dummy_head->next(0)); + } + + const_iterator cbegin() const { + return const_iterator(dummy_head->next(0)); + } + + iterator end() { + return iterator(nullptr); + } + + const_iterator end() const { + return const_iterator(nullptr); + } + + const_iterator cend() const { + return const_iterator(nullptr); + } + + size_type size() const { + return my_size.load(std::memory_order_relaxed); + } + + size_type max_size() const { + return my_node_allocator.max_size(); + } + + bool empty() const { + return 0 == size(); + } + + allocator_type get_allocator() const { + return my_node_allocator; + } + + void swap(concurrent_skip_list& other) { + using std::swap; + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + tbb::internal::allocator_swap(my_node_allocator, other.my_node_allocator, pocs_type()); + swap(my_compare, other.my_compare); + swap(my_rnd_generator, other.my_rnd_generator); + swap(dummy_head, other.dummy_head); + + size_type tmp = my_size; + my_size.store(other.my_size); + other.my_size.store(tmp); + } + + std::pair equal_range(const key_type& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + std::pair equal_range(const key_type& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + key_compare key_comp() const { return my_compare; } + + value_compare value_comp() const { return traits_type::value_comp(my_compare); } + + class const_range_type : tbb::internal::no_assign { + public: + using size_type = typename concurrent_skip_list::size_type; + using value_type = typename concurrent_skip_list::value_type; + using iterator = typename concurrent_skip_list::const_iterator; + private: + const_iterator my_end; + const_iterator my_begin; + size_type my_level; + + public: + + bool empty() const { + return my_begin.my_node_ptr->next(0) == my_end.my_node_ptr; + } + + bool is_divisible() const { + return my_level != 0 ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr : false; + } + + size_type size() const { return std::distance(my_begin, my_end);} + + const_range_type( const_range_type& r, split) + : my_end(r.my_end) { + my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); + my_level = my_begin.my_node_ptr->height(); + r.my_end = my_begin; + } + + const_range_type( const concurrent_skip_list& l) + : my_end(l.end()), my_begin(l.begin()), my_level(my_begin.my_node_ptr->height() ) {} + + iterator begin() const { return my_begin; } + iterator end() const { return my_end; } + size_t grainsize() const { return 1; } + + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_skip_list::iterator; + + range_type(range_type& r, split) : const_range_type(r, split()) {} + range_type(const concurrent_skip_list& l) : const_range_type(l) {} + + iterator begin() const { + node_ptr node = const_range_type::begin().my_node_ptr; + return iterator(node); + } + + iterator end() const { + node_ptr node = const_range_type::end().my_node_ptr; + return iterator(node); } + }; // class range_type + + range_type range() { return range_type(*this); } + const_range_type range() const { return const_range_type(*this); } + +private: + void internal_move(concurrent_skip_list&& other) { + dummy_head = other.dummy_head; + other.dummy_head = nullptr; + other.create_dummy_head(); + + my_size = other.my_size.load(); + other.my_size = 0; + } + + static const key_type& get_key(node_ptr n) { + __TBB_ASSERT(n, NULL); + return traits_type::get_key(n->value()); + } + + template + iterator internal_find(const K& key) { + iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + const_iterator internal_find(const K& key) const { + const_iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + std::pair range = equal_range(key); + return std::distance(range.first, range.second); + } + return (find(key) == end()) ? size_type(0) : size_type(1); + } + + /** + * Finds position on the @param level using @param cmp + * @param level - on which level search prev node + * @param prev - pointer to the start node to search + * @param key - key to search + * @param cmp - callable object to compare two objects + * (my_compare member is default comparator) + * @returns pointer to the node which is not satisfy the comparison with @param key + */ + template + pointer_type internal_find_position( size_type level, pointer_type& prev, const K& key, + const comparator& cmp) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + pointer_type curr = prev->next(level); + + while (curr && cmp(get_key(curr), key)) { + prev = curr; + __TBB_ASSERT(level < prev->height(), NULL); + curr = prev->next(level); + } + + return curr; + } + + template + void fill_prev_next_arrays(array_type& prev_nodes, array_type& next_nodes, node_ptr prev, const key_type& key, + const comparator& cmp) { + prev_nodes.fill(dummy_head); + next_nodes.fill(nullptr); + + for (size_type h = prev->height(); h > 0; --h) { + node_ptr next = internal_find_position(h - 1, prev, key, cmp); + prev_nodes[h - 1] = prev; + next_nodes[h - 1] = next; + } + } + + template + std::pair internal_insert(Args&&... args) { + node_ptr new_node = create_node(std::forward(args)...); + std::pair insert_result = internal_insert_node(new_node); + if(!insert_result.second) { + delete_node(new_node); + } + return insert_result; + } + + std::pair internal_insert_node(node_ptr new_node) { + array_type prev_nodes; + array_type next_nodes; + __TBB_ASSERT(dummy_head->height() >= new_node->height(), "Wrong height for new node"); + + do { + if (allow_multimapping) { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), + not_greater_compare(my_compare)); + } else { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), my_compare); + } + + node_ptr next = next_nodes[0]; + if (next && !allow_multimapping && !my_compare(get_key(new_node), get_key(next))) { + // TODO: do we really need to wait? + while (!next->fully_linked()) { + // TODO: atomic backoff + } + + return std::pair(iterator(next), false); + } + __TBB_ASSERT(allow_multimapping || !next || my_compare(get_key(new_node), get_key(next)), + "Wrong elements order"); + + } while (!try_insert_node(new_node, prev_nodes, next_nodes)); + + __TBB_ASSERT(new_node, NULL); + return std::pair(iterator(new_node), true); + } + + bool try_insert_node(node_ptr new_node, array_type& prev_nodes, array_type& next_nodes) { + __TBB_ASSERT(dummy_head->height() >= new_node->height(), NULL); + + lock_array locks; + + if (!try_lock_nodes(new_node->height(), prev_nodes, next_nodes, locks)) { + return false; + } + + __TBB_ASSERT(allow_multimapping || + ((prev_nodes[0] == dummy_head || + my_compare(get_key(prev_nodes[0]), get_key(new_node))) && + (next_nodes[0] == nullptr || my_compare(get_key(new_node), get_key(next_nodes[0])))), + "Wrong elements order"); + + for (size_type level = 0; level < new_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(prev_nodes[level]->next(level) == next_nodes[level], NULL); + new_node->set_next(level, next_nodes[level]); + prev_nodes[level]->set_next(level, new_node); + } + new_node->mark_linked(); + + ++my_size; + + return true; + } + + bool try_lock_nodes(size_type height, array_type& prevs, array_type& next_nodes, lock_array& locks) { + for (size_type l = 0; l < height; ++l) { + if (l == 0 || prevs[l] != prevs[l - 1]) + locks[l] = prevs[l]->acquire(); + + node_ptr next = prevs[l]->next(l); + if ( next != next_nodes[l]) return false; + } + + return true; + } + + template + const_iterator internal_get_bound(const K& key, const comparator& cmp) const { + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return const_iterator(next); + } + + template + iterator internal_get_bound(const K& key, const comparator& cmp){ + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return iterator(next); + } + + // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted + std::pair internal_extract(const_iterator it) { + if ( it != end() ) { + key_type key = traits_type::get_key(*it); + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + array_type prev_nodes; + array_type next_nodes; + + fill_prev_next_arrays(prev_nodes, next_nodes, prev, key, my_compare); + + node_ptr erase_node = next_nodes[0]; + node_ptr next_node = erase_node->next(0); + + if (erase_node && !my_compare(key, get_key(erase_node))) { + for(size_type level = 0; level < erase_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(next_nodes[level] == erase_node, NULL); + prev_nodes[level]->set_next(level, erase_node->next(level)); + } + --my_size; + return std::pair(erase_node, next_node); + } + } + return std::pair(nullptr, nullptr); + } + +protected: + template + void internal_merge(SourceType&& source) { + using source_type = typename std::decay::type; + using source_iterator = typename source_type::iterator; + __TBB_STATIC_ASSERT((std::is_same::value), "Incompatible containers cannot be merged"); + + for(source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || !contains(traits_type::get_key(*where))) { + std::pair extract_result = source.internal_extract(where); + + //If the insertion fails - return the node into source + node_type handle(extract_result.first); + __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); + + if (!insert(std::move(handle)).second) { + source.insert(std::move(handle)); + } + handle.deactivate(); + } + } + } + +private: + void internal_copy(const concurrent_skip_list& other) { + internal_copy(other.begin(), other.end()); + } + + template + void internal_copy(Iterator first, Iterator last) { + clear(); + try { + for (auto it = first; it != last; ++it) + insert(*it); + } + catch (...) { + clear(); + delete_dummy_head(); + throw; + } + } + + /** Generate random level */ + size_type random_level() { + return my_rnd_generator(); + } + + static size_type calc_node_size(size_type height) { + return sizeof(list_node_type) + height*sizeof(typename list_node_type::atomic_node_pointer); + } + + /** Creates new node */ + template + node_ptr create_node(Args&&... args) { + size_type levels = random_level(); + + size_type sz = calc_node_size(levels); + + node_ptr node = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + + try { + node_allocator_traits::construct(my_node_allocator, node, levels); + + } + catch(...) { + deallocate_node(node, sz); + throw; + } + + try { + node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); + } + catch (...) { + node_allocator_traits::destroy(my_node_allocator, node); + deallocate_node(node, sz); + throw; + } + + return node; + } + + void create_dummy_head() { + size_type sz = calc_node_size(MAX_LEVEL); + + dummy_head = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + // TODO: investigate linkage fail in debug without this workaround + auto max_level = MAX_LEVEL; + + try { + node_allocator_traits::construct(my_node_allocator, dummy_head, max_level); + } + catch(...) { + deallocate_node(dummy_head, sz); + throw; + } + } + + template + void delete_node(node_ptr node) { + size_type sz = calc_node_size(node->height()); + // Destroy value + if (!is_dummy) node_allocator_traits::destroy(my_node_allocator, node->storage()); + // Destroy node + node_allocator_traits::destroy(my_node_allocator, node); + // Deallocate memory + deallocate_node(node, sz); + } + + void deallocate_node(node_ptr node, size_type sz) { + node_allocator_traits::deallocate(my_node_allocator, reinterpret_cast(node), sz); + } + + void delete_dummy_head() { + delete_node(dummy_head); + } + + static iterator get_iterator(const_iterator it) { + return iterator(it.my_node_ptr); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::true_type) { + delete_dummy_head(); + tbb::internal::allocator_move_assignment(my_node_allocator, other.my_node_allocator, std::true_type()); + internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::false_type) { + if (my_node_allocator == other.my_node_allocator) { + delete_dummy_head(); + internal_move(std::move(other)); + } else { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + struct not_greater_compare { + const key_compare& my_less_compare; + + not_greater_compare(const key_compare& less_compare) : my_less_compare(less_compare) {} + + template + bool operator()(const K1& first, const K2& second) const { + return !my_less_compare(second, first); + } + }; + + node_allocator_type my_node_allocator; + key_compare my_compare; + random_level_generator_type my_rnd_generator; + node_ptr dummy_head; + + template + friend class concurrent_skip_list; + + std::atomic my_size; +}; // class concurrent_skip_list + +template +class concurrent_geometric_level_generator { +public: + static constexpr size_t max_level = MAX_LEVEL; + + concurrent_geometric_level_generator() : engines(time(NULL)) {} + + size_t operator()() { + return (distribution(engines.local()) % MAX_LEVEL) + 1; + } + +private: + tbb::enumerable_thread_specific engines; + std::geometric_distribution distribution; +}; + +} // namespace internal +} // namespace interface10 +} // namespace tbb + +#endif // __TBB_concurrent_skip_list_H diff --git a/inst/include/tbb_local/tbb/internal/_concurrent_unordered_impl.h b/inst/include/tbb_local/tbb/internal/_concurrent_unordered_impl.h index 211a5c3d..3998c3fa 100644 --- a/inst/include/tbb_local/tbb/internal/_concurrent_unordered_impl.h +++ b/inst/include/tbb_local/tbb/internal/_concurrent_unordered_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -29,12 +25,6 @@ #include "../tbb_stddef.h" -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include #include // Need std::pair #include // Need std::equal_to (in ../concurrent_unordered_*.h) @@ -42,10 +32,6 @@ #include // Need std::memset #include __TBB_STD_SWAP_HEADER -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - #include "../atomic.h" #include "../tbb_exception.h" #include "../tbb_allocator.h" @@ -54,7 +40,17 @@ #include #endif +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_COPY_DELETION_BROKEN + #define __TBB_UNORDERED_NODE_HANDLE_PRESENT 1 +#endif + +#include "_allocator_traits.h" #include "_tbb_hash_compare_impl.h" +#include "_template_helpers.h" + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT +#include "_node_handle_impl.h" +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT namespace tbb { namespace interface5 { @@ -134,6 +130,8 @@ class solist_iterator : public flist_iterator friend class split_ordered_list; template friend class solist_iterator; + template + friend class concurrent_unordered_base; template friend bool operator==( const solist_iterator &i, const solist_iterator &j ); template @@ -195,17 +193,20 @@ class split_ordered_list { public: typedef split_ordered_list self_type; - typedef typename Allocator::template rebind::other allocator_type; + + typedef typename tbb::internal::allocator_rebind::type allocator_type; + struct node; typedef node *nodeptr_t; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::value_type value_type; + typedef typename tbb::internal::allocator_traits::value_type value_type; + typedef typename tbb::internal::allocator_traits::size_type size_type; + typedef typename tbb::internal::allocator_traits::difference_type difference_type; + typedef typename tbb::internal::allocator_traits::pointer pointer; + typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; + // No support for reference/const_reference in allocator traits + typedef value_type& reference; + typedef const value_type& const_reference; typedef solist_iterator const_iterator; typedef solist_iterator iterator; @@ -230,6 +231,15 @@ class split_ordered_list return my_order_key; } + // get() and value() is a common interface for getting access to node`s element (required by node_handle) + value_type* storage() { + return reinterpret_cast(&my_element); + } + + value_type& value() { + return *storage(); + } + // Inserts the new element in the list in an atomic fashion nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) { @@ -574,27 +584,49 @@ class split_ordered_list } - // This erase function can handle both real and dummy nodes - void erase_node(raw_iterator previous, raw_const_iterator& where) - { + nodeptr_t erase_node_impl(raw_iterator previous, raw_const_iterator& where) { nodeptr_t pnode = (where++).get_node_ptr(); nodeptr_t prevnode = previous.get_node_ptr(); __TBB_ASSERT(prevnode->my_next == pnode, "Erase must take consecutive iterators"); prevnode->my_next = pnode->my_next; + return pnode; + } + // This erase function can handle both real and dummy nodes + void erase_node(raw_iterator previous, raw_const_iterator& where, + /*allow_destroy*/tbb::internal::true_type) + { + nodeptr_t pnode = erase_node_impl(previous, where); destroy_node(pnode); } + void erase_node(raw_iterator previous, raw_const_iterator& where, + /*allow_destroy*/tbb::internal::false_type) + { + erase_node_impl(previous, where); + } + + void erase_node(raw_iterator previous, raw_const_iterator& where) { + erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); + } + // Erase the element (previous node needs to be passed because this is a forward only list) - iterator erase_node(raw_iterator previous, const_iterator where) + template + iterator erase_node(raw_iterator previous, const_iterator where, AllowDestroy) { raw_const_iterator it = where; - erase_node(previous, it); + erase_node(previous, it, AllowDestroy()); my_element_count--; return get_iterator(first_real_iterator(it)); } + iterator erase_node(raw_iterator previous, const_iterator& where) { + return erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); + } + + + // Move all elements from the passed in split-ordered list to this one void move_all(self_type& source) { @@ -649,7 +681,7 @@ class split_ordered_list #endif } - typename allocator_type::template rebind::other my_node_allocator; // allocator object for nodes + typename tbb::internal::allocator_rebind::type my_node_allocator; // allocator object for nodes size_type my_element_count; // Total item count, not counting dummy nodes nodeptr_t my_head; // pointer to head node }; @@ -671,12 +703,15 @@ class concurrent_unordered_base : public Traits typedef typename Traits::allocator_type allocator_type; typedef typename hash_compare::hasher hasher; typedef typename hash_compare::key_equal key_equal; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; + + typedef typename tbb::internal::allocator_traits::size_type size_type; + typedef typename tbb::internal::allocator_traits::difference_type difference_type; + typedef typename tbb::internal::allocator_traits::pointer pointer; + typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; + // No support for reference/const_reference in allocator + typedef typename allocator_type::value_type& reference; + typedef const typename allocator_type::value_type& const_reference; + typedef split_ordered_list solist_t; typedef typename solist_t::nodeptr_t nodeptr_t; // Iterators that walk the entire split-order list, including dummy nodes @@ -686,12 +721,19 @@ class concurrent_unordered_base : public Traits typedef typename solist_t::const_iterator const_iterator; typedef iterator local_iterator; typedef const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename Traits::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT using Traits::my_hash_compare; using Traits::get_key; using Traits::allow_multimapping; static const size_type initial_bucket_number = 8; // Initial number of buckets + private: + template + friend class concurrent_unordered_base; + typedef std::pair pairii_t; typedef std::pair paircc_t; @@ -737,8 +779,10 @@ class concurrent_unordered_base : public Traits #if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_base(concurrent_unordered_base&& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) + : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()), + my_maximum_bucket_size(float(initial_bucket_load)) { + my_number_of_buckets = initial_bucket_number; internal_init(); swap(right); } @@ -750,6 +794,8 @@ class concurrent_unordered_base : public Traits internal_init(); if (a == right.get_allocator()){ + my_number_of_buckets = initial_bucket_number; + my_maximum_bucket_size = float(initial_bucket_load); this->swap(right); }else{ my_maximum_bucket_size = right.my_maximum_bucket_size; @@ -830,6 +876,43 @@ class concurrent_unordered_base : public Traits internal_clear(); } +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void internal_merge(SourceType& source) { + typedef typename SourceType::iterator source_iterator; + __TBB_STATIC_ASSERT((tbb::internal::is_same_type::value), + "Incompatible containers cannot be merged"); + + for(source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || find(get_key(*where)) == end()) { + std::pair extract_result = source.internal_extract(where); + + // If the insertion fails, it returns ownership of the node to extract_result.first + // extract_result.first remains valid node handle + if (!insert(std::move(extract_result.first)).second) { + raw_iterator next = extract_result.second; + raw_iterator current = next++; + + __TBB_ASSERT(extract_result.first.my_node->get_order_key() >= current.get_node_ptr()->get_order_key(), + "Wrong nodes order in source container"); + __TBB_ASSERT(next==source.my_solist.raw_end() || + extract_result.first.my_node->get_order_key() <= next.get_node_ptr()->get_order_key(), + "Wrong nodes order in source container"); + + size_t new_count = 0;// To use try_insert() + bool insert_result = + source.my_solist.try_insert(current, next, extract_result.first.my_node, &new_count).second; + __TBB_ASSERT_EX(insert_result, "Return to source must be successful. " + "Changing source container while merging is unsafe."); + } + extract_result.first.deactivate(); + } + } + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + public: allocator_type get_allocator() const { return my_solist.get_allocator(); @@ -966,7 +1049,8 @@ class concurrent_unordered_base : public Traits // Modifiers std::pair insert(const value_type& value) { - return internal_insert(value); + return internal_insert(value); } iterator insert(const_iterator, const value_type& value) { @@ -976,23 +1060,43 @@ class concurrent_unordered_base : public Traits #if __TBB_CPP11_RVALUE_REF_PRESENT std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); + return internal_insert(std::move(value)); } iterator insert(const_iterator, value_type&& value) { // Ignore hint return insert(std::move(value)).first; } +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + std::pair insert(node_type&& nh) { + if (!nh.empty()) { + nodeptr_t handled_node = nh.my_node; + std::pair insert_result = + internal_insert + (handled_node->my_element, handled_node); + if (insert_result.second) + nh.deactivate(); + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert(const_iterator, node_type&& nh) { + return insert(std::move(nh)).first; + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT template std::pair emplace(Args&&... args) { nodeptr_t pnode = my_solist.create_node_v(tbb::internal::forward(args)...); - const sokey_t hashed_element_key = (sokey_t) my_hash_compare(get_key(pnode->my_element)); - const sokey_t order_key = split_order_key_regular(hashed_element_key); - pnode->init(order_key); - return internal_insert(pnode->my_element, pnode); + return internal_insert(pnode->my_element, pnode); } template @@ -1000,9 +1104,8 @@ class concurrent_unordered_base : public Traits // Ignore hint return emplace(tbb::internal::forward(args)...).first; } +#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif // __TBB_CPP11_RVALUE_REF_PRESENT template void insert(Iterator first, Iterator last) { @@ -1034,9 +1137,21 @@ class concurrent_unordered_base : public Traits return item_count; } +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + node_type unsafe_extract(const_iterator where) { + return internal_extract(where).first; + } + + node_type unsafe_extract(const key_type& key) { + pairii_t where = equal_range(key); + if (where.first == end()) return node_type(); // element was not found + return internal_extract(where.first).first; + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + void swap(concurrent_unordered_base& right) { if (this != &right) { - std::swap(my_hash_compare, right.my_hash_compare); // TODO: check what ADL meant here + std::swap(my_hash_compare, right.my_hash_compare); my_solist.swap(right.my_solist); internal_swap_buckets(right); std::swap(my_number_of_buckets, right.my_number_of_buckets); @@ -1209,8 +1324,8 @@ class concurrent_unordered_base : public Traits // Initialize the hash and keep the first bucket open void internal_init() { - // Allocate an array of segment pointers - memset((void*) my_buckets, 0, pointers_per_table * sizeof(void *)); + // Initialize the array of segment pointers + memset(my_buckets, 0, sizeof(my_buckets)); // Initialize bucket 0 raw_iterator dummy_node = my_solist.raw_begin(); @@ -1268,7 +1383,7 @@ class concurrent_unordered_base : public Traits } // Insert an element in the hash given its value - template + template std::pair internal_insert(__TBB_FORWARDING_REF(ValueType) value, nodeptr_t pnode = NULL) { const key_type *pkey = &get_key(value); @@ -1293,6 +1408,11 @@ class concurrent_unordered_base : public Traits // If the value was moved, the known reference to key might be invalid pkey = &get_key(pnode->my_element); } + else + { + // Set new order_key to node + pnode->init(order_key); + } // Try to insert 'pnode' between 'previous' and 'where' std::pair result = my_solist.try_insert(previous, where, pnode, &new_count); @@ -1317,7 +1437,7 @@ class concurrent_unordered_base : public Traits else if (!allow_multimapping && solist_t::get_order_key(where) == order_key && !my_hash_compare(get_key(*where), *pkey)) // TODO: fix negation { // Element already in the list, return it - if (pnode) + if (pnode && AllowDestroy::value) my_solist.destroy_node(pnode); return std::pair(my_solist.get_iterator(where), false); } @@ -1363,14 +1483,33 @@ class concurrent_unordered_base : public Traits __TBB_ASSERT(previous != last, "Invalid head node"); // First node is a dummy node - for (raw_iterator where = previous; ; previous = where) { + for (raw_iterator where = previous; where != last; previous = where) { ++where; - if (where == last) - return end(); - else if (my_solist.get_iterator(where) == it) + if (my_solist.get_iterator(where) == it) return my_solist.erase_node(previous, it); } + return end(); + } + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + std::pair internal_extract(const_iterator it) { + sokey_t hash_key = sokey_t(my_hash_compare(get_key(*it))); + raw_iterator previous = prepare_bucket(hash_key); + raw_iterator last = my_solist.raw_end(); + __TBB_ASSERT(previous != last, "Invalid head node"); + + for(raw_iterator where = previous; where != last; previous = where) { + ++where; + if (my_solist.get_iterator(where) == it) { + const_iterator result = it; + my_solist.erase_node(previous, it, /*allow_destroy*/tbb::internal::false_type()); + return std::pair( node_type(result.get_node_ptr()), + previous); + } + } + return std::pair(node_type(), end()); } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Return the [begin, end) pair of iterators with the same key values. // This operation makes sense only if mapping is many-to-one. @@ -1478,7 +1617,7 @@ class concurrent_unordered_base : public Traits if (my_buckets[segment] == NULL) { size_type sz = segment_size(segment); raw_iterator * new_segment = my_allocator.allocate(sz); - std::memset((void*) new_segment, 0, sz*sizeof(raw_iterator)); + std::memset(static_cast(new_segment), 0, sz*sizeof(raw_iterator)); if (my_buckets[segment].compare_and_swap( new_segment, NULL) != NULL) my_allocator.deallocate(new_segment, sz); @@ -1513,7 +1652,7 @@ class concurrent_unordered_base : public Traits // Shared variables atomic my_number_of_buckets; // Current table size solist_t my_solist; // List where all the elements are kept - typename allocator_type::template rebind::other my_allocator; // Allocator object for segments + typename tbb::internal::allocator_rebind::type my_allocator; // Allocator object for segments float my_maximum_bucket_size; // Maximum size of the bucket atomic my_buckets[pointers_per_table]; // The segment table }; diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_async_msg_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_async_msg_impl.h index 7d2c3aef..ffb63ce4 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_async_msg_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_async_msg_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_async_msg_impl_H @@ -25,101 +21,28 @@ #error Do not #include this internal file directly; use public TBB headers instead. #endif -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -template< typename T > class async_msg; - namespace internal { -template< typename T, typename = void > -struct async_helpers { - typedef async_msg async_type; - typedef T filtered_type; - - static const bool is_async_type = false; - - static const void* to_void_ptr(const T& t) { - return static_cast(&t); - } - - static void* to_void_ptr(T& t) { - return static_cast(&t); - } - - static const T& from_void_ptr(const void* p) { - return *static_cast(p); - } - - static T& from_void_ptr(void* p) { - return *static_cast(p); - } - - static task* try_put_task_wrapper_impl( receiver* const this_recv, const void *p, bool is_async ) { - if ( is_async ) { - // This (T) is NOT async and incoming 'A t' IS async - // Get data from async_msg - const async_msg& msg = async_helpers< async_msg >::from_void_ptr(p); - task* const new_task = msg.my_storage->subscribe(*this_recv); - // finalize() must be called after subscribe() because set() can be called in finalize() - // and 'this_recv' client must be subscribed by this moment - msg.finalize(); - return new_task; - } else { - // Incoming 't' is NOT async - return this_recv->try_put_task( from_void_ptr(p) ); - } - } -}; - -template< typename T > -struct async_helpers< T, typename std::enable_if< std::is_base_of, T>::value >::type > { - typedef T async_type; - typedef typename T::async_msg_data_type filtered_type; - - static const bool is_async_type = true; - - // Receiver-classes use const interfaces - static const void* to_void_ptr(const T& t) { - return static_cast( &static_cast&>(t) ); - } - - static void* to_void_ptr(T& t) { - return static_cast( &static_cast&>(t) ); - } - - // Sender-classes use non-const interfaces - static const T& from_void_ptr(const void* p) { - return *static_cast( static_cast*>(p) ); - } - - static T& from_void_ptr(void* p) { - return *static_cast( static_cast*>(p) ); - } - - // Used in receiver class - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if ( is_async ) { - // Both are async - return this_recv->try_put_task( from_void_ptr(p) ); - } else { - // This (T) is async and incoming 'X t' is NOT async - // Create async_msg for X - const filtered_type& t = async_helpers::from_void_ptr(p); - const T msg(t); - return this_recv->try_put_task(msg); - } - } -}; - template class async_storage { public: typedef receiver async_storage_client; - async_storage() { my_data_ready.store(false); } + async_storage() : my_graph(nullptr) { + my_data_ready.store(false); + } + + ~async_storage() { + // Release reference to the graph if async_storage + // was destructed before set() call + if (my_graph) { + my_graph->release_wait(); + my_graph = nullptr; + } + } template - async_storage(C&& data) : my_data( std::forward(data) ) { + async_storage(C&& data) : my_graph(nullptr), my_data( std::forward(data) ) { using namespace tbb::internal; __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); @@ -148,10 +71,16 @@ class async_storage { (*it)->try_put(my_data); } + // Data was sent, release reference to the graph + if (my_graph) { + my_graph->release_wait(); + my_graph = nullptr; + } + return true; } - task* subscribe(async_storage_client& client) { + task* subscribe(async_storage_client& client, graph& g) { if (! my_data_ready.load()) { tbb::spin_mutex::scoped_lock locker(my_mutex); @@ -163,6 +92,10 @@ class async_storage { } #endif // TBB_USE_ASSERT + // Increase graph lifetime + my_graph = &g; + my_graph->reserve_wait(); + // Subscribe my_clients.push_back(&client); return SUCCESSFULLY_ENQUEUED; @@ -174,11 +107,10 @@ class async_storage { } private: + graph* my_graph; tbb::spin_mutex my_mutex; - tbb::atomic my_data_ready; T my_data; - typedef std::vector subscriber_list_type; subscriber_list_type my_clients; }; diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_body_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_body_impl.h index 54f37411..57264fd3 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_body_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_body_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_body_impl_H @@ -33,11 +29,46 @@ typedef tbb::internal::uint64_t tag_value; using tbb::internal::strip; +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + +template struct Policy {}; + +template struct has_policy; + +template +struct has_policy : + tbb::internal::bool_constant::value || + has_policy::value> {}; + +template +struct has_policy : + tbb::internal::bool_constant::value> {}; + +template +struct has_policy > : has_policy {}; + +#else + +template struct Policy {}; + +template +struct has_policy : tbb::internal::bool_constant::value> {}; + +template +struct has_policy > : has_policy {}; + +template +struct has_policy > : + tbb::internal::bool_constant::value || has_policy::value> {}; + +#endif + namespace graph_policy_namespace { struct rejecting { }; struct reserving { }; struct queueing { }; + struct lightweight { }; // K == type of field used for key-matching. Each tag-matching port will be provided // functor that, given an object accepted by the port, will return the @@ -52,6 +83,10 @@ namespace graph_policy_namespace { // old tag_matching join's new specifier typedef key_matching tag_matching; + // Aliases for Policy combinations + typedef interface10::internal::Policy queueing_lightweight; + typedef interface10::internal::Policy rejecting_lightweight; + } // namespace graph_policy_namespace // -------------- function_body containers ---------------------- @@ -232,13 +267,20 @@ class type_to_key_function_body_leaf : public type_to_key_funct //! A task that calls a node's forward_task function template< typename NodeType > -class forward_task_bypass : public task { +class forward_task_bypass : public graph_task { NodeType &my_node; public: - forward_task_bypass( NodeType &n ) : my_node(n) {} + forward_task_bypass( NodeType &n +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_node(n) {} task *execute() __TBB_override { task * new_task = my_node.forward_task(); @@ -250,14 +292,21 @@ class forward_task_bypass : public task { //! A task that calls a node's apply_body_bypass function, passing in an input of type Input // return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL template< typename NodeType, typename Input > -class apply_body_task_bypass : public task { +class apply_body_task_bypass : public graph_task { NodeType &my_node; Input my_input; public: - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} + apply_body_task_bypass( NodeType &n, const Input &i +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_node(n), my_input(i) {} task *execute() __TBB_override { task * next_task = my_node.apply_body_bypass( my_input ); @@ -268,7 +317,7 @@ class apply_body_task_bypass : public task { //! A task that calls a node's apply_body_bypass function with no input template< typename NodeType > -class source_task_bypass : public task { +class source_task_bypass : public graph_task { NodeType &my_node; @@ -291,13 +340,85 @@ struct empty_body { Output operator()( const Input & ) const { return Output(); } }; +template +class decrementer; + +template +class decrementer::value, void>::type + > : public receiver, tbb::internal::no_copy { + T* my_node; +protected: + + task* try_put_task( const DecrementType& value ) __TBB_override { + task* result = my_node->decrement_counter( value ); + if( !result ) + result = SUCCESSFULLY_ENQUEUED; + return result; + } + + graph& graph_reference() __TBB_override { + return my_node->my_graph; + } + + template friend class tbb::flow::interface11::limiter_node; + void reset_receiver( reset_flags f ) __TBB_override { +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + if (f & rf_clear_edges) + my_built_predecessors.clear(); +#else + tbb::internal::suppress_unused_warning( f ); +#endif + } + +public: + // Since decrementer does not make use of possibly unconstructed owner inside its + // constructor, my_node can be directly initialized with 'this' pointer passed from the + // owner, hence making method 'set_owner' needless. + decrementer() : my_node(NULL) {} + void set_owner( T *node ) { my_node = node; } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + spin_mutex my_mutex; + //! The predecessor type for this node + typedef typename receiver::predecessor_type predecessor_type; + + typedef internal::edge_container built_predecessors_type; + typedef typename built_predecessors_type::edge_list_type predecessor_list_type; + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + + void internal_add_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.add_edge( s ); + } + + void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.delete_edge(s); + } + + void copy_predecessors( predecessor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.copy_edges(v); + } + + size_t predecessor_count() __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + return my_built_predecessors.edge_count(); + } +protected: + built_predecessors_type my_built_predecessors; +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ +}; + template -class decrementer : public continue_receiver, tbb::internal::no_copy { +class decrementer : public continue_receiver, tbb::internal::no_copy { T *my_node; task *execute() __TBB_override { - return my_node->decrement_counter(); + return my_node->decrement_counter( 1 ); } protected: @@ -310,7 +431,15 @@ class decrementer : public continue_receiver, tbb::internal::no_copy { typedef continue_msg input_type; typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } + decrementer( int number_of_predecessors = 0 ) + : continue_receiver( + __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, tbb::flow::internal::no_priority) + ) + // Since decrementer does not make use of possibly unconstructed owner inside its + // constructor, my_node can be directly initialized with 'this' pointer passed from the + // owner, hence making method 'set_owner' needless. + , my_node(NULL) + {} void set_owner( T *node ) { my_node = node; } }; diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_cache_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_cache_impl.h index 7fac84e8..f3081b6f 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_cache_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_cache_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_cache_impl_H @@ -57,12 +53,12 @@ class node_cache { void clear() { while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_predecessors.clear(); #endif } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef edge_container built_predecessors_type; built_predecessors_type &built_predecessors() { return my_built_predecessors; } @@ -86,14 +82,14 @@ class node_cache { typename mutex_type::scoped_lock lock(my_mutex); return (size_t)(my_built_predecessors.edge_count()); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: typedef M mutex_type; mutex_type my_mutex; std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type my_built_predecessors; #endif @@ -189,7 +185,7 @@ class predecessor_cache : public node_cache< sender, M > { protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION using node_cache< predecessor_type, M >::my_built_predecessors; #endif successor_type *my_owner; @@ -290,7 +286,7 @@ class successor_cache : tbb::internal::no_copy { typedef sender owner_type; #endif // __TBB_PREVIEW_ASYNC_MSG typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_successors; #endif successors_type my_successors; @@ -298,7 +294,7 @@ class successor_cache : tbb::internal::no_copy { owner_type *my_owner; public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename edge_container::edge_list_type successor_list_type; edge_container &built_successors() { return my_built_successors; } @@ -323,7 +319,7 @@ class successor_cache : tbb::internal::no_copy { return my_built_successors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ successor_cache( ) : my_owner(NULL) {} @@ -354,7 +350,7 @@ class successor_cache : tbb::internal::no_copy { void clear() { my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_successors.clear(); #endif } @@ -381,7 +377,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { #endif // __TBB_PREVIEW_ASYNC_MSG typedef std::list< pointer_type > successors_type; successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_successors; typedef edge_container::edge_list_type successor_list_type; #endif @@ -390,7 +386,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container &built_successors() { return my_built_successors; } @@ -414,7 +410,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { return my_built_successors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ successor_cache( ) : my_owner(NULL) {} @@ -452,7 +448,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { void clear() { my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_successors.clear(); #endif } @@ -508,6 +504,40 @@ class broadcast_cache : public successor_cache { return last_task; } + // call try_put_task and return list of received tasks +#if __TBB_PREVIEW_ASYNC_MSG + template + bool gather_successful_try_puts( const X &t, task_list &tasks ) { +#else + bool gather_successful_try_puts( const T &t, task_list &tasks ) { +#endif // __TBB_PREVIEW_ASYNC_MSG + bool upgraded = true; + bool is_at_least_one_put_successful = false; + typename mutex_type::scoped_lock l(this->my_mutex, upgraded); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + task * new_task = (*i)->try_put_task(t); + if(new_task) { + ++i; + if(new_task != SUCCESSFULLY_ENQUEUED) { + tasks.push_back(*new_task); + } + is_at_least_one_put_successful = true; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + if (!upgraded) { + l.upgrade_to_writer(); + upgraded = true; + } + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return is_at_least_one_put_successful; + } }; //! A cache of successors that are put in a round-robin fashion diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_impl.h index 92278caf..a4b96d10 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,828 +12,508 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +#ifndef __TBB_flow_graph_impl_H +#define __TBB_flow_graph_impl_H +#include "../tbb_stddef.h" +#include "../task.h" +#include "../task_arena.h" +#include "../flow_graph_abstractions.h" +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#include "../concurrent_priority_queue.h" +#endif -*/ - -#ifndef __TBB__flow_graph_impl_H -#define __TBB__flow_graph_impl_H +#include -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. +#if TBB_DEPRECATED_FLOW_ENQUEUE +#define FLOW_SPAWN(a) tbb::task::enqueue((a)) +#else +#define FLOW_SPAWN(a) tbb::task::spawn((a)) #endif -// included in namespace tbb::flow::interfaceX (in flow_graph.h) +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) expr +#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) , priority +#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1, priority +#else +#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) +#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) +#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1 +#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) expr +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1, arg2 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg3, arg4 +#else +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg2 +#endif // TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + +namespace tbb { +namespace flow { namespace internal { +static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +typedef unsigned int node_priority_t; +static const node_priority_t no_priority = node_priority_t(0); +#endif +} - typedef tbb::internal::uint64_t tag_value; +namespace interface10 { - using tbb::internal::strip; +using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; - namespace graph_policy_namespace { +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +using tbb::flow::internal::node_priority_t; +using tbb::flow::internal::no_priority; +//! Base class for tasks generated by graph nodes. +struct graph_task : public task { + graph_task( node_priority_t node_priority = no_priority ) : priority( node_priority ) {} + node_priority_t priority; +}; +#else +typedef task graph_task; +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ - struct rejecting { }; - struct reserving { }; - struct queueing { }; - // K == type of field used for key-matching. Each tag-matching port will be provided - // functor that, given an object accepted by the port, will return the - /// field of type K being used for matching. - template::type > > - struct key_matching { - typedef K key_type; - typedef typename strip::type base_key_type; - typedef KHash hash_compare_type; - }; +class graph; +class graph_node; - // old tag_matching join's new specifier - typedef key_matching tag_matching; +template +class graph_iterator { + friend class graph; + friend class graph_node; +public: + typedef size_t size_type; + typedef GraphNodeType value_type; + typedef GraphNodeType* pointer; + typedef GraphNodeType& reference; + typedef const GraphNodeType& const_reference; + typedef std::forward_iterator_tag iterator_category; + + //! Default constructor + graph_iterator() : my_graph(NULL), current_node(NULL) {} + + //! Copy constructor + graph_iterator(const graph_iterator& other) : + my_graph(other.my_graph), current_node(other.current_node) + {} + + //! Assignment + graph_iterator& operator=(const graph_iterator& other) { + if (this != &other) { + my_graph = other.my_graph; + current_node = other.current_node; + } + return *this; } -// -------------- function_body containers ---------------------- - - //! A functor that takes no input and generates a value of type Output - template< typename Output > - class source_body : tbb::internal::no_assign { - public: - virtual ~source_body() {} - virtual bool operator()(Output &output) = 0; - virtual source_body* clone() = 0; - }; - - //! The leaf for source_body - template< typename Output, typename Body> - class source_body_leaf : public source_body { - public: - source_body_leaf( const Body &_body ) : body(_body) { } - bool operator()(Output &output) __TBB_override { return body( output ); } - source_body_leaf* clone() __TBB_override { - return new source_body_leaf< Output, Body >(body); - } - Body get_body() { return body; } - private: - Body body; - }; - - //! A functor that takes an Input and generates an Output - template< typename Input, typename Output > - class function_body : tbb::internal::no_assign { - public: - virtual ~function_body() {} - virtual Output operator()(const Input &input) = 0; - virtual function_body* clone() = 0; - }; - - //! the leaf for function_body - template - class function_body_leaf : public function_body< Input, Output > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, Output, B >(body); - } - private: - B body; - }; - - //! the leaf for function_body specialized for Input and output of continue_msg - template - class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()( const continue_msg &i ) __TBB_override { - body(i); - return i; - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, continue_msg, B >(body); - } - private: - B body; - }; + //! Dereference + reference operator*() const; - //! the leaf for function_body specialized for Output of continue_msg - template - class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()(const Input &i) __TBB_override { - body(i); - return continue_msg(); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< Input, continue_msg, B >(body); - } - private: - B body; - }; + //! Dereference + pointer operator->() const; - //! the leaf for function_body specialized for Input of continue_msg - template - class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { - public: - function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const continue_msg &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - function_body_leaf* clone() __TBB_override { - return new function_body_leaf< continue_msg, Output, B >(body); - } - private: - B body; - }; - - //! function_body that takes an Input and a set of output ports - template - class multifunction_body : tbb::internal::no_assign { - public: - virtual ~multifunction_body () {} - virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; - virtual multifunction_body* clone() = 0; - virtual void* get_body_ptr() = 0; - }; + //! Equality + bool operator==(const graph_iterator& other) const { + return ((my_graph == other.my_graph) && (current_node == other.current_node)); + } - //! leaf for multifunction. OutputSet can be a std::tuple or a vector. - template - class multifunction_body_leaf : public multifunction_body { - public: - multifunction_body_leaf(const B &_body) : body(_body) { } - void operator()(const Input &input, OutputSet &oset) __TBB_override { - body(input, oset); // body may explicitly put() to one or more of oset. - } - void* get_body_ptr() __TBB_override { return &body; } - multifunction_body_leaf* clone() __TBB_override { - return new multifunction_body_leaf(body); - } + //! Inequality + bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } - private: - B body; - }; + //! Pre-increment + graph_iterator& operator++() { + internal_forward(); + return *this; + } -// ------ function bodies for hash_buffers and key-matching joins. + //! Post-increment + graph_iterator operator++(int) { + graph_iterator result = *this; + operator++(); + return result; + } -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual Output operator()(const Input &input) = 0; // returns an Output - virtual type_to_key_function_body* clone() = 0; +private: + // the graph over which we are iterating + GraphContainerType *my_graph; + // pointer into my_graph's my_nodes list + pointer current_node; + + //! Private initializing constructor for begin() and end() iterators + graph_iterator(GraphContainerType *g, bool begin); + void internal_forward(); +}; // class graph_iterator + +// flags to modify the behavior of the graph reset(). Can be combined. +enum reset_flags { + rf_reset_protocol = 0, + rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body. + rf_clear_edges = 1 << 1 // delete edges }; -// specialization for ref output -template -class type_to_key_function_body : tbb::internal::no_assign { - public: - virtual ~type_to_key_function_body() {} - virtual const Output & operator()(const Input &input) = 0; // returns a const Output& - virtual type_to_key_function_body* clone() = 0; -}; +namespace internal { -template -class type_to_key_function_body_leaf : public type_to_key_function_body { -public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output, B>(body); +void activate_graph(graph& g); +void deactivate_graph(graph& g); +bool is_graph_active(graph& g); +tbb::task& prioritize_task(graph& g, tbb::task& arena_task); +void spawn_in_graph_arena(graph& g, tbb::task& arena_task); +void enqueue_in_graph_arena(graph &g, tbb::task& arena_task); +void add_task_to_graph_reset_list(graph& g, tbb::task *tp); + +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +struct graph_task_comparator { + bool operator()(const graph_task* left, const graph_task* right) { + return left->priority < right->priority; } -private: - B body; }; -template -class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { +typedef tbb::concurrent_priority_queue graph_task_priority_queue_t; + +class priority_task_selector : public task { public: - type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - const Output& operator()(const Input &i) __TBB_override { - return body(i); - } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { - return new type_to_key_function_body_leaf< Input, Output&, B>(body); + priority_task_selector(graph_task_priority_queue_t& priority_queue) + : my_priority_queue(priority_queue) {} + task* execute() __TBB_override { + graph_task* t = NULL; + bool result = my_priority_queue.try_pop(t); + __TBB_ASSERT_EX( result, "Number of critical tasks for scheduler and tasks" + " in graph's priority queue mismatched" ); + __TBB_ASSERT( t && t != SUCCESSFULLY_ENQUEUED, + "Incorrect task submitted to graph priority queue" ); + __TBB_ASSERT( t->priority != tbb::flow::internal::no_priority, + "Tasks from graph's priority queue must have priority" ); + task* t_next = t->execute(); + task::destroy(*t); + return t_next; } private: - B body; + graph_task_priority_queue_t& my_priority_queue; }; +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ -// --------------------------- end of function_body containers ------------------------ - -// --------------------------- node task bodies --------------------------------------- - - //! A task that calls a node's forward_task function - template< typename NodeType > - class forward_task_bypass : public task { +} - NodeType &my_node; +//! The graph class +/** This class serves as a handle to the graph */ +class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { + friend class graph_node; + template< typename Body > + class run_task : public graph_task { public: - - forward_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task * new_task = my_node.forward_task(); - if (new_task == SUCCESSFULLY_ENQUEUED) new_task = NULL; - return new_task; + run_task(Body& body +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_body(body) { } + tbb::task *execute() __TBB_override { + my_body(); + return NULL; } + private: + Body my_body; }; - //! A task that calls a node's apply_body_bypass function, passing in an input of type Input - // return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL - template< typename NodeType, typename Input > - class apply_body_task_bypass : public task { - - NodeType &my_node; - Input my_input; - + template< typename Receiver, typename Body > + class run_and_put_task : public graph_task { public: - - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} - - task *execute() __TBB_override { - task * next_task = my_node.apply_body_bypass( my_input ); - if(next_task == SUCCESSFULLY_ENQUEUED) next_task = NULL; - return next_task; + run_and_put_task(Receiver &r, Body& body) : my_receiver(r), my_body(body) {} + tbb::task *execute() __TBB_override { + tbb::task *res = my_receiver.try_put_task(my_body()); + if (res == SUCCESSFULLY_ENQUEUED) res = NULL; + return res; } + private: + Receiver &my_receiver; + Body my_body; }; + typedef std::list task_list_type; - //! A task that calls a node's apply_body_bypass function with no input - template< typename NodeType > - class source_task_bypass : public task { - - NodeType &my_node; - + class wait_functor { + tbb::task* graph_root_task; public: - - source_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task *new_task = my_node.apply_body_bypass( ); - if(new_task == SUCCESSFULLY_ENQUEUED) return NULL; - return new_task; - } - }; - -// ------------------------ end of node task bodies ----------------------------------- - - //! An empty functor that takes an Input and returns a default constructed Output - template< typename Input, typename Output > - struct empty_body { - Output operator()( const Input & ) const { return Output(); } + wait_functor(tbb::task* t) : graph_root_task(t) {} + void operator()() const { graph_root_task->wait_for_all(); } }; - //! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. - template< typename T, typename M=spin_mutex > - class node_cache { - public: - - typedef size_t size_type; - - bool empty() { - typename mutex_type::scoped_lock lock( my_mutex ); - return internal_empty(); - } - - void add( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - internal_push(n); - } - - void remove( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - for ( size_t i = internal_size(); i != 0; --i ) { - T &s = internal_pop(); - if ( &s == &n ) return; // only remove one predecessor per request - internal_push(s); - } - } - - void clear() { - while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef edge_container built_predecessors_type; - built_predecessors_type &built_predecessors() { return my_built_predecessors; } - - typedef typename edge_container::edge_list_type predecessor_list_type; - void internal_add_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.add_edge(n); - } - - void internal_delete_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.delete_edge(n); - } - - void copy_predecessors( predecessor_list_type &v) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() { - typename mutex_type::scoped_lock lock(my_mutex); - return (size_t)(my_built_predecessors.edge_count()); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - protected: - - typedef M mutex_type; - mutex_type my_mutex; - std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type my_built_predecessors; -#endif - - // Assumes lock is held - inline bool internal_empty( ) { - return my_q.empty(); - } - - // Assumes lock is held - inline size_type internal_size( ) { - return my_q.size(); - } - - // Assumes lock is held - inline void internal_push( T &n ) { - my_q.push(&n); - } - - // Assumes lock is held - inline T &internal_pop() { - T *v = my_q.front(); - my_q.pop(); - return *v; - } - - }; - - //! A cache of predecessors that only supports try_get - template< typename T, typename M=spin_mutex > -#if __TBB_PREVIEW_ASYNC_MSG - // TODO: make predecessor_cache type T-independent when async_msg becomes regular feature - class predecessor_cache : public node_cache< untyped_sender, M > { -#else - class predecessor_cache : public node_cache< sender, M > { -#endif // __TBB_PREVIEW_ASYNC_MSG + //! A functor that spawns a task + class spawn_functor : tbb::internal::no_assign { + tbb::task& spawn_task; public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - predecessor_cache( ) : my_owner( NULL ) { } - - void set_owner( successor_type *owner ) { my_owner = owner; } - - bool get_item( output_type &v ) { - - bool msg = false; - - do { - predecessor_type *src; - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( this->internal_empty() ) { - break; - } - src = &this->internal_pop(); - } - - // Try to get from this sender - msg = src->try_get( v ); - - if (msg == false) { - // Relinquish ownership of the edge - if (my_owner) - src->register_successor( *my_owner ); - } else { - // Retain ownership of the edge - this->add(*src); - } - } while ( msg == false ); - return msg; + spawn_functor(tbb::task& t) : spawn_task(t) {} + void operator()() const { + FLOW_SPAWN(spawn_task); } - - // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). - void reset() { - if (my_owner) { - for(;;) { - predecessor_type *src; - { - if (this->internal_empty()) break; - src = &this->internal_pop(); - } - src->register_successor( *my_owner ); - } - } - } - - protected: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - using node_cache< predecessor_type, M >::my_built_predecessors; -#endif - successor_type *my_owner; }; - //! An cache of predecessors that supports requests and reservations - // TODO: make reservable_predecessor_cache type T-independent when async_msg becomes regular feature - template< typename T, typename M=spin_mutex > - class reservable_predecessor_cache : public predecessor_cache< T, M > { - public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - reservable_predecessor_cache( ) : reserved_src(NULL) { } - - bool - try_reserve( output_type &v ) { - bool msg = false; - - do { - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( reserved_src || this->internal_empty() ) - return false; - - reserved_src = &this->internal_pop(); - } - - // Try to get from this sender - msg = reserved_src->try_reserve( v ); - - if (msg == false) { - typename mutex_type::scoped_lock lock(this->my_mutex); - // Relinquish ownership of the edge - reserved_src->register_successor( *this->my_owner ); - reserved_src = NULL; - } else { - // Retain ownership of the edge - this->add( *reserved_src ); - } - } while ( msg == false ); - - return msg; + void prepare_task_arena(bool reinit = false) { + if (reinit) { + __TBB_ASSERT(my_task_arena, "task arena is NULL"); + my_task_arena->terminate(); + my_task_arena->initialize(tbb::task_arena::attach()); } - - bool - try_release( ) { - reserved_src->try_release( ); - reserved_src = NULL; - return true; - } - - bool - try_consume( ) { - reserved_src->try_consume( ); - reserved_src = NULL; - return true; - } - - void reset( ) { - reserved_src = NULL; - predecessor_cache::reset( ); - } - - void clear() { - reserved_src = NULL; - predecessor_cache::clear(); + else { + __TBB_ASSERT(my_task_arena == NULL, "task arena is not NULL"); + my_task_arena = new tbb::task_arena(tbb::task_arena::attach()); } + if (!my_task_arena->is_active()) // failed to attach + my_task_arena->initialize(); // create a new, default-initialized arena + __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); + } - private: - predecessor_type *reserved_src; - }; - +public: + //! Constructs a graph with isolated task_group_context + graph(); - //! An abstract cache of successors - // TODO: make successor_cache type T-independent when async_msg becomes regular feature - template - class successor_cache : tbb::internal::no_copy { - protected: + //! Constructs a graph with use_this_context as context + explicit graph(tbb::task_group_context& use_this_context); - typedef M mutex_type; - mutex_type my_mutex; + //! Destroys the graph. + /** Calls wait_for_all, then destroys the root task and context. */ + ~graph(); -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; - typedef untyped_sender owner_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; - typedef sender owner_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + void set_name(const char *name); #endif - successors_type my_successors; - - owner_type *my_owner; - public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename edge_container::edge_list_type successor_list_type; - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( owner_type *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} + void increment_wait_count() { + reserve_wait(); + } - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - } + void decrement_wait_count() { + release_wait(); + } - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( typename successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - my_successors.erase(i); - break; - } - } + //! Used to register that an external entity may still interact with the graph. + /** The graph will not return from wait_for_all until a matching number of decrement_wait_count calls + is made. */ + void reserve_wait() __TBB_override; + + //! Deregisters an external entity that may have interacted with the graph. + /** The graph will not return from wait_for_all until all the number of decrement_wait_count calls + matches the number of increment_wait_count calls. */ + void release_wait() __TBB_override; + + //! Spawns a task that runs a body and puts its output to a specific receiver + /** The task is spawned as a child of the graph. This is useful for running tasks + that need to block a wait_for_all() on the graph. For example a one-off source. */ + template< typename Receiver, typename Body > + void run(Receiver &r, Body body) { + if (internal::is_graph_active(*this)) { + task* rtask = new (task::allocate_additional_child_of(*root_task())) + run_and_put_task< Receiver, Body >(r, body); + my_task_arena->execute(spawn_functor(*rtask)); } + } - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); + //! Spawns a task that runs a function object + /** The task is spawned as a child of the graph. This is useful for running tasks + that need to block a wait_for_all() on the graph. For example a one-off source. */ + template< typename Body > + void run(Body body) { + if (internal::is_graph_active(*this)) { + task* rtask = new (task::allocate_additional_child_of(*root_task())) run_task< Body >(body); + my_task_arena->execute(spawn_functor(*rtask)); } + } - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); + //! Wait until graph is idle and decrement_wait_count calls equals increment_wait_count calls. + /** The waiting thread will go off and steal work while it is block in the wait_for_all. */ + void wait_for_all() { + cancelled = false; + caught_exception = false; + if (my_root_task) { +#if TBB_USE_EXCEPTIONS + try { #endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const T &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - }; // successor_cache - - //! An abstract cache of successors, specialized to continue_msg - template<> - class successor_cache< continue_msg > : tbb::internal::no_copy { - protected: - - typedef spin_rw_mutex mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; - successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - edge_container my_built_successors; - typedef edge_container::edge_list_type successor_list_type; + my_task_arena->execute(wait_functor(my_root_task)); + cancelled = my_context->is_group_execution_cancelled(); +#if TBB_USE_EXCEPTIONS + } + catch (...) { + my_root_task->set_ref_count(1); + my_context->reset(); + caught_exception = true; + cancelled = true; + throw; + } #endif - - sender *my_owner; - - public: - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); + // TODO: the "if" condition below is just a work-around to support the concurrent wait + // mode. The cancellation and exception mechanisms are still broken in this mode. + // Consider using task group not to re-implement the same functionality. + if (!(my_context->traits() & tbb::task_group_context::concurrent_wait)) { + my_context->reset(); // consistent with behavior in catch() + my_root_task->set_ref_count(1); + } } + } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - successor_cache( ) : my_owner(NULL) {} + //! Returns the root task of the graph + tbb::task * root_task() { + return my_root_task; + } - void set_owner( sender *owner ) { my_owner = owner; } + // ITERATORS + template + friend class graph_iterator; + + // Graph iterator typedefs + typedef graph_iterator iterator; + typedef graph_iterator const_iterator; + + // Graph iterator constructors + //! start iterator + iterator begin(); + //! end iterator + iterator end(); + //! start const iterator + const_iterator begin() const; + //! end const iterator + const_iterator end() const; + //! start const iterator + const_iterator cbegin() const; + //! end const iterator + const_iterator cend() const; + + //! return status of graph execution + bool is_cancelled() { return cancelled; } + bool exception_thrown() { return caught_exception; } + + // thread-unsafe state reset. + void reset(reset_flags f = rf_reset_protocol); - virtual ~successor_cache() {} +private: + tbb::task *my_root_task; + tbb::task_group_context *my_context; + bool own_context; + bool cancelled; + bool caught_exception; + bool my_is_active; + task_list_type my_reset_task_list; - void register_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - if ( my_owner && r.is_continue_receiver() ) { - r.register_predecessor( *my_owner ); - } - } + graph_node *my_nodes, *my_nodes_last; - void remove_successor( successor_type &r ) { - mutex_type::scoped_lock l(my_mutex, true); - for ( successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - // TODO: Check if we need to test for continue_receiver before - // removing from r. - if ( my_owner ) - r.remove_predecessor( *my_owner ); - my_successors.erase(i); - break; - } - } - } + tbb::spin_mutex nodelist_mutex; + void register_node(graph_node *n); + void remove_node(graph_node *n); - bool empty() { - mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } + tbb::task_arena* my_task_arena; - void clear() { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_successors.clear(); +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + internal::graph_task_priority_queue_t my_priority_queue; #endif - } -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const continue_msg &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - - }; // successor_cache< continue_msg > - - //! A cache of successors that are broadcast to - // TODO: make broadcast_cache type T-independent when async_msg becomes regular feature - template - class broadcast_cache : public successor_cache { - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; + friend void internal::activate_graph(graph& g); + friend void internal::deactivate_graph(graph& g); + friend bool internal::is_graph_active(graph& g); + friend tbb::task& internal::prioritize_task(graph& g, tbb::task& arena_task); + friend void internal::spawn_in_graph_arena(graph& g, tbb::task& arena_task); + friend void internal::enqueue_in_graph_arena(graph &g, tbb::task& arena_task); + friend void internal::add_task_to_graph_reset_list(graph& g, tbb::task *tp); + + friend class tbb::interface7::internal::task_arena_base; + +}; // class graph + +//! The base of all graph nodes. +class graph_node : tbb::internal::no_copy { + friend class graph; + template + friend class graph_iterator; +protected: + graph& my_graph; + graph_node *next, *prev; +public: + explicit graph_node(graph& g); - public: + virtual ~graph_node(); - broadcast_cache( ) {} +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + virtual void set_name(const char *name) = 0; +#endif - // as above, but call try_put_task instead, and return the last task we received (if any) -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task * try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - task * last_task = NULL; - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - last_task = combine_tasks(last_task, new_task); // enqueue if necessary - if(new_task) { - ++i; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return last_task; - } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + virtual void extract() = 0; +#endif - }; +protected: + // performs the reset on an individual node. + virtual void reset_node(reset_flags f = rf_reset_protocol) = 0; +}; // class graph_node - //! A cache of successors that are put in a round-robin fashion - // TODO: make round_robin_cache type T-independent when async_msg becomes regular feature - template - class round_robin_cache : public successor_cache { - typedef size_t size_type; - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; +namespace internal { - public: +inline void activate_graph(graph& g) { + g.my_is_active = true; +} - round_robin_cache( ) {} +inline void deactivate_graph(graph& g) { + g.my_is_active = false; +} - size_type size() { - typename mutex_type::scoped_lock l(this->my_mutex, false); - return this->my_successors.size(); - } +inline bool is_graph_active(graph& g) { + return g.my_is_active; +} -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +inline tbb::task& prioritize_task(graph& g, tbb::task& t) { + task* critical_task = &t; + // TODO: change flow graph's interfaces to work with graph_task type instead of tbb::task. + graph_task* gt = static_cast(&t); + if( gt->priority != no_priority ) { + //! Non-preemptive priority pattern. The original task is submitted as a work item to the + //! priority queue, and a new critical task is created to take and execute a work item with + //! the highest known priority. The reference counting responsibility is transferred (via + //! allocate_continuation) to the new task. + critical_task = new( gt->allocate_continuation() ) priority_task_selector(g.my_priority_queue); + tbb::internal::make_critical( *critical_task ); + g.my_priority_queue.push(gt); + } + return *critical_task; +} #else - task *try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - if ( new_task ) { - return new_task; - } else { - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } - else { - ++i; - } - } - } - return NULL; - } - }; - - template - class decrementer : public continue_receiver, tbb::internal::no_copy { - - T *my_node; - - task *execute() __TBB_override { - return my_node->decrement_counter(); - } - - public: +inline tbb::task& prioritize_task(graph&, tbb::task& t) { + return t; +} +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ + +//! Spawns a task inside graph arena +inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { + if (is_graph_active(g)) { + graph::spawn_functor s_fn(prioritize_task(g, arena_task)); + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); + g.my_task_arena->execute(s_fn); + } +} - typedef continue_msg input_type; - typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } - void set_owner( T *node ) { my_node = node; } - }; +//! Enqueues a task inside graph arena +inline void enqueue_in_graph_arena(graph &g, tbb::task& arena_task) { + if (is_graph_active(g)) { + __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); + task::enqueue(prioritize_task(g, arena_task), *g.my_task_arena); + } +} +inline void add_task_to_graph_reset_list(graph& g, tbb::task *tp) { + g.my_reset_task_list.push_back(tp); } -#endif // __TBB__flow_graph_impl_H +} // namespace internal + +} // namespace interface10 +} // namespace flow +} // namespace tbb +#endif // __TBB_flow_graph_impl_H diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_indexer_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_indexer_impl.h index 1fc6690c..332467dd 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_indexer_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_indexer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_indexer_impl_H @@ -43,18 +39,18 @@ namespace internal { template struct indexer_helper { template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p) { + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { typedef typename tuple_element::type T; task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get(my_input).set_up(p, indexer_node_put_task); - indexer_helper::template set_indexer_node_pointer(my_input, p); + tbb::flow::get(my_input).set_up(p, indexer_node_put_task, g); + indexer_helper::template set_indexer_node_pointer(my_input, p, g); } template static inline void reset_inputs(InputTuple &my_input, reset_flags f) { indexer_helper::reset_inputs(my_input, f); tbb::flow::get(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract(InputTuple &my_input) { indexer_helper::extract(my_input); @@ -66,16 +62,16 @@ namespace internal { template struct indexer_helper { template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p) { + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { typedef typename tuple_element<0, TupleTypes>::type T; task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get<0>(my_input).set_up(p, indexer_node_put_task); + tbb::flow::get<0>(my_input).set_up(p, indexer_node_put_task, g); } template static inline void reset_inputs(InputTuple &my_input, reset_flags f) { tbb::flow::get<0>(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract(InputTuple &my_input) { tbb::flow::get<0>(my_input).extract_receiver(); @@ -89,22 +85,24 @@ namespace internal { void* my_indexer_ptr; typedef task* (* forward_function_ptr)(T const &, void* ); forward_function_ptr my_try_put_task; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION spin_mutex my_pred_mutex; typedef typename receiver::built_predecessors_type built_predecessors_type; built_predecessors_type my_built_predecessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + graph* my_graph; public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - indexer_input_port() : my_pred_mutex() {} - indexer_input_port( const indexer_input_port & /*other*/ ) : receiver(), my_pred_mutex() { +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + indexer_input_port() : my_pred_mutex(), my_graph(NULL) {} + indexer_input_port( const indexer_input_port & other) : receiver(), my_pred_mutex(), my_graph(other.my_graph) { } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - void set_up(void *p, forward_function_ptr f) { - my_indexer_ptr = p; - my_try_put_task = f; - } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + void set_up(void* p, forward_function_ptr f, graph& g) { + my_indexer_ptr = p; + my_try_put_task = f; + my_graph = &g; + } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename receiver::predecessor_type predecessor_type; @@ -126,7 +124,7 @@ namespace internal { spin_mutex::scoped_lock l(my_pred_mutex); my_built_predecessors.copy_edges(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template< typename R, typename B > friend class run_and_put_task; template friend class internal::broadcast_cache; @@ -135,14 +133,18 @@ namespace internal { return my_try_put_task(v, my_indexer_ptr); } + graph& graph_reference() __TBB_override { + return *my_graph; + } + public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void reset_receiver(reset_flags f) __TBB_override { if(f&rf_clear_edges) my_built_predecessors.clear(); } #else void reset_receiver(reset_flags /*f*/) __TBB_override { } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract_receiver() { my_built_predecessors.receiver_extract(*this); } #endif }; @@ -154,7 +156,7 @@ namespace internal { typedef OutputType output_type; typedef InputTuple input_type; - // Some versions of Intel C++ compiler fail to generate an implicit constructor for the class which has std::tuple as a member. + // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member. indexer_node_FE() : my_inputs() {} input_type &input_ports() { return my_inputs; } @@ -174,7 +176,7 @@ namespace internal { typedef StructTypes tuple_types; typedef typename sender::successor_type successor_type; typedef indexer_node_FE input_ports_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -182,7 +184,7 @@ namespace internal { private: // ----------- Aggregator ------------ enum op_type { reg_succ, rem_succ, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy #endif @@ -196,7 +198,7 @@ namespace internal { output_type const *my_arg; successor_type *my_succ; task *bypass_t; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; successor_list_type *succv; #endif @@ -233,7 +235,7 @@ namespace internal { __TBB_store_with_release(current->status, SUCCEEDED); // return of try_put_task actual return value } break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_succ: my_successors.internal_add_built_successor(*(current->my_succ)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -250,20 +252,20 @@ namespace internal { my_successors.copy_successors(*(current->succv)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } // ---------- end aggregator ----------- public: indexer_node_base(graph& g) : graph_node(g), input_ports_type() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this); + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g); my_successors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); } indexer_node_base(const indexer_node_base& other) : graph_node(other.my_graph), input_ports_type(), sender() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this); + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph); my_successors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); } @@ -286,7 +288,7 @@ namespace internal { return op_data.bypass_t; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -315,7 +317,7 @@ namespace internal { my_successors.built_successors().sender_extract(*this); indexer_helper::extract(this->my_inputs); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: void reset_node(reset_flags f) __TBB_override { if(f & rf_clear_edges) { diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_item_buffer_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_item_buffer_impl.h index 85d2686d..9ac4dbbb 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_item_buffer_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_item_buffer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_item_buffer_impl_H @@ -144,7 +140,7 @@ namespace internal { return get_my_item(my_tail - 1); } - // following methods are for reservation of the front of a bufffer. + // following methods are for reservation of the front of a buffer. void reserve_item(size_type i) { __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); item(i).second = reserved_item; } void release_item(size_type i) { __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); item(i).second = has_item; } diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_join_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_join_impl.h index 4999bef7..4ccaef9f 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_join_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_join_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_join_impl_H @@ -27,21 +23,21 @@ namespace internal { - struct forwarding_base { - forwarding_base(graph &g) : graph_pointer(&g) {} + struct forwarding_base : tbb::internal::no_assign { + forwarding_base(graph &g) : graph_ref(g) {} virtual ~forwarding_base() {} // decrement_port_count may create a forwarding task. If we cannot handle the task // ourselves, ask decrement_port_count to deal with it. virtual task * decrement_port_count(bool handle_task) = 0; virtual void increment_port_count() = 0; // moved here so input ports can queue tasks - graph* graph_pointer; + graph& graph_ref; }; // specialization that lets us keep a copy of the current_key for building results. // KeyType can be a reference type. template - struct matching_forwarding_base :public forwarding_base { + struct matching_forwarding_base : public forwarding_base { typedef typename tbb::internal::strip::type current_key_type; matching_forwarding_base(graph &g) : forwarding_base(g) { } virtual task * increment_key_count(current_key_type const & /*t*/, bool /*handle_task*/) = 0; // {return NULL;} @@ -126,7 +122,7 @@ namespace internal { tbb::flow::get(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract_inputs(InputTuple &my_input) { join_helper::extract_inputs(my_input); @@ -200,7 +196,7 @@ namespace internal { tbb::flow::get<0>(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract_inputs(InputTuple &my_input) { tbb::flow::get<0>(my_input).extract_receiver(); @@ -214,18 +210,17 @@ namespace internal { public: typedef T input_type; typedef typename receiver::predecessor_type predecessor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename receiver::built_predecessors_type built_predecessors_type; #endif private: // ----------- Aggregator ------------ enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef reserving_port class_type; class reserving_port_operation : public aggregated_operation { @@ -234,7 +229,7 @@ namespace internal { union { T *my_arg; predecessor_type *my_pred; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; predecessor_list_type *plist; #endif @@ -294,7 +289,7 @@ namespace internal { my_predecessors.try_consume( ); __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_predecessors.internal_add_built_predecessor(*(current->my_pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -311,7 +306,7 @@ namespace internal { my_predecessors.copy_predecessors(*(current->plist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -324,6 +319,10 @@ namespace internal { return NULL; } + graph& graph_reference() __TBB_override { + return my_join->graph_ref; + } + public: //! Constructor @@ -378,7 +377,7 @@ namespace internal { my_aggregator.execute(&op_data); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } void internal_add_built_predecessor(predecessor_type &src) __TBB_override { reserving_port_operation op_data(src, add_blt_pred); @@ -406,7 +405,7 @@ namespace internal { my_predecessors.built_predecessors().receiver_extract(*this); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ void reset_receiver( reset_flags f) __TBB_override { if(f & rf_clear_edges) my_predecessors.clear(); @@ -429,7 +428,7 @@ namespace internal { typedef T input_type; typedef typename receiver::predecessor_type predecessor_type; typedef queueing_port class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif @@ -437,18 +436,17 @@ namespace internal { // ----------- Aggregator ------------ private: enum op_type { get__item, res_port, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class queueing_port_operation : public aggregated_operation { public: char type; T my_val; T *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION predecessor_type *pred; size_t cnt_val; predecessor_list_type *plist; @@ -509,7 +507,7 @@ namespace internal { } __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_built_predecessors.add_edge(*(current->pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -526,7 +524,7 @@ namespace internal { my_built_predecessors.copy_edges(*(current->plist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -544,6 +542,10 @@ namespace internal { return op_data.bypass_t; } + graph& graph_reference() __TBB_override { + return my_join->graph_ref; + } + public: //! Constructor @@ -577,7 +579,7 @@ namespace internal { return; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } void internal_add_built_predecessor(predecessor_type &p) __TBB_override { @@ -608,12 +610,12 @@ namespace internal { item_buffer::reset(); my_built_predecessors.receiver_extract(*this); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ void reset_receiver(reset_flags f) __TBB_override { tbb::internal::suppress_unused_warning(f); item_buffer::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION if (f & rf_clear_edges) my_built_predecessors.clear(); #endif @@ -621,7 +623,7 @@ namespace internal { private: forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_predecessors; #endif }; // queueing_port @@ -659,7 +661,7 @@ namespace internal { typedef typename TraitsType::TtoK type_to_key_func_type; typedef typename TraitsType::KHash hash_compare_type; typedef hash_buffer< key_type, input_type, type_to_key_func_type, hash_compare_type > buffer_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif @@ -667,18 +669,17 @@ namespace internal { // ----------- Aggregator ------------ private: enum op_type { try__put, get__item, res_port -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class key_matching_port_operation : public aggregated_operation { public: char type; input_type my_val; input_type *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION predecessor_type *pred; size_t cnt_val; predecessor_list_type *plist; @@ -721,7 +722,7 @@ namespace internal { this->delete_with_key(my_join->current_key); __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_built_predecessors.add_edge(*(current->pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -759,6 +760,10 @@ namespace internal { return rtask; } + graph& graph_reference() __TBB_override { + return my_join->graph_ref; + } + public: key_matching_port() : receiver(), buffer_type() { @@ -789,7 +794,7 @@ namespace internal { return op_data.status == SUCCEEDED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } void internal_add_built_predecessor(predecessor_type &p) __TBB_override { @@ -825,7 +830,7 @@ namespace internal { return; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract_receiver() { buffer_type::reset(); my_built_predecessors.receiver_extract(*this); @@ -834,7 +839,7 @@ namespace internal { void reset_receiver(reset_flags f ) __TBB_override { tbb::internal::suppress_unused_warning(f); buffer_type::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION if (f & rf_clear_edges) my_built_predecessors.clear(); #endif @@ -844,7 +849,7 @@ namespace internal { // my_join forwarding base used to count number of inputs that // received key. matching_forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_predecessors; #endif }; // key_matching_port @@ -871,7 +876,7 @@ namespace internal { join_helper::set_join_node_pointer(my_inputs, this); } - join_node_FE(const join_node_FE& other) : forwarding_base(*(other.forwarding_base::graph_pointer)), my_node(NULL) { + join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { ports_with_no_inputs = N; join_helper::set_join_node_pointer(my_inputs, this); } @@ -885,11 +890,11 @@ namespace internal { // if all input_ports have predecessors, spawn forward to try and consume tuples task * decrement_port_count(bool handle_task) __TBB_override { if(ports_with_no_inputs.fetch_and_decrement() == 1) { - if(this->graph_pointer->is_active()) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) + if(internal::is_graph_active(this->graph_ref)) { + task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) forward_task_bypass(*my_node); if(!handle_task) return rtask; - FLOW_SPAWN(*rtask); + internal::spawn_in_graph_arena(this->graph_ref, *rtask); } } return NULL; @@ -905,7 +910,7 @@ namespace internal { join_helper::reset_inputs(my_inputs, f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) { // called outside of parallel contexts ports_with_no_inputs = N; @@ -949,7 +954,7 @@ namespace internal { join_helper::set_join_node_pointer(my_inputs, this); } - join_node_FE(const join_node_FE& other) : forwarding_base(*(other.forwarding_base::graph_pointer)), my_node(NULL) { + join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { ports_with_no_items = N; join_helper::set_join_node_pointer(my_inputs, this); } @@ -965,11 +970,11 @@ namespace internal { task * decrement_port_count(bool handle_task) __TBB_override { if(ports_with_no_items.fetch_and_decrement() == 1) { - if(this->graph_pointer->is_active()) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) + if(internal::is_graph_active(this->graph_ref)) { + task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) forward_task_bypass (*my_node); if(!handle_task) return rtask; - FLOW_SPAWN( *rtask); + internal::spawn_in_graph_arena(this->graph_ref, *rtask); } } return NULL; @@ -986,7 +991,7 @@ namespace internal { join_helper::reset_inputs(my_inputs, f ); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { reset_port_count(); join_helper::extract_inputs(my_inputs); @@ -1055,7 +1060,6 @@ namespace internal { // and the output_buffer_type base class private: enum op_type { res_count, inc_count, may_succeed, try_make }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_FE, InputTuple, OutputTuple> class_type; class key_matching_FE_operation : public aggregated_operation { @@ -1084,16 +1088,16 @@ namespace internal { task * fill_output_buffer(unref_key_type &t, bool should_enqueue, bool handle_task) { output_type l_out; task *rtask = NULL; - bool do_fwd = should_enqueue && this->buffer_empty() && this->graph_pointer->is_active(); + bool do_fwd = should_enqueue && this->buffer_empty() && internal::is_graph_active(this->graph_ref); this->current_key = t; this->delete_with_key(this->current_key); // remove the key if(join_helper::get_items(my_inputs, l_out)) { // <== call back this->push_back(l_out); if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item - rtask = new ( task::allocate_additional_child_of( *(this->graph_pointer->root_task()) ) ) + rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) forward_task_bypass(*my_node); if(handle_task) { - FLOW_SPAWN(*rtask); + internal::spawn_in_graph_arena(this->graph_ref, *rtask); rtask = NULL; } do_fwd = false; @@ -1167,7 +1171,7 @@ namespace internal { this->set_key_func(cfb); } - join_node_FE(const join_node_FE& other) : forwarding_base_type(*(other.forwarding_base_type::graph_pointer)), key_to_count_buffer_type(), + join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(), output_buffer_type() { my_node = NULL; join_helper::set_join_node_pointer(my_inputs, this); @@ -1210,7 +1214,7 @@ namespace internal { output_buffer_type::reset(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { // called outside of parallel contexts join_helper::extract_inputs(my_inputs); @@ -1262,7 +1266,7 @@ namespace internal { using input_ports_type::try_to_make_tuple; using input_ports_type::tuple_accepted; using input_ports_type::tuple_rejected; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -1270,11 +1274,10 @@ namespace internal { private: // ----------- Aggregator ------------ enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_base class_type; class join_node_base_operation : public aggregated_operation { @@ -1283,7 +1286,7 @@ namespace internal { union { output_type *my_arg; successor_type *my_succ; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; successor_list_type *slist; #endif @@ -1309,11 +1312,11 @@ namespace internal { switch(current->type) { case reg_succ: { my_successors.register_successor(*(current->my_succ)); - if(tuple_build_may_succeed() && !forwarder_busy && this->graph_node::my_graph.is_active()) { - task *rtask = new ( task::allocate_additional_child_of(*(this->graph_node::my_graph.root_task())) ) + if(tuple_build_may_succeed() && !forwarder_busy && internal::is_graph_active(my_graph)) { + task *rtask = new ( task::allocate_additional_child_of(*(my_graph.root_task())) ) forward_task_bypass >(*this); - FLOW_SPAWN(*rtask); + internal::spawn_in_graph_arena(my_graph, *rtask); forwarder_busy = true; } __TBB_store_with_release(current->status, SUCCEEDED); @@ -1342,7 +1345,7 @@ namespace internal { build_succeeded = try_to_make_tuple(out); // fetch front_end of queue if(build_succeeded) { task *new_task = my_successors.try_put_task(out); - last_task = combine_tasks(last_task, new_task); + last_task = combine_tasks(my_graph, last_task, new_task); if(new_task) { tuple_accepted(); } @@ -1358,7 +1361,7 @@ namespace internal { forwarder_busy = false; } break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_succ: my_successors.internal_add_built_successor(*(current->my_succ)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -1375,7 +1378,7 @@ namespace internal { my_successors.copy_successors(*(current->slist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -1420,7 +1423,7 @@ namespace internal { return op_data.status == SUCCEEDED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } void internal_add_built_successor( successor_type &r) __TBB_override { @@ -1444,9 +1447,9 @@ namespace internal { op_data.slist = &l; my_aggregator.execute(&op_data); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { input_ports_type::extract(); my_successors.built_successors().sender_extract(*this); diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_node_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_node_impl.h index eeff72bc..215777f3 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_node_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_node_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_node_impl_H @@ -61,15 +57,15 @@ namespace internal { //! Input and scheduling for a function node that takes a type Input as input // The only up-ref is apply_body_impl, which should implement the function // call and any handling of the result. - template< typename Input, typename A, typename ImplType > + template< typename Input, typename Policy, typename A, typename ImplType > class function_input_base : public receiver, tbb::internal::no_assign { - enum op_type {reg_pred, rem_pred, app_body, try_fwd, tryput_bypass, app_body_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy // create vector copies of preds and succs #endif }; - typedef function_input_base class_type; + typedef function_input_base class_type; public: @@ -79,25 +75,32 @@ namespace internal { typedef predecessor_cache predecessor_cache_type; typedef function_input_queue input_queue_type; typedef typename A::template rebind< input_queue_type >::other queue_allocator_type; + __TBB_STATIC_ASSERT(!((internal::has_policy::value) && (internal::has_policy::value)), + "queueing and rejecting policies can't be specified simultaneously"); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif //! Constructor for function_input_base - function_input_base( graph &g, size_t max_concurrency, input_queue_type *q = NULL) - : my_graph_ptr(&g), my_max_concurrency(max_concurrency), my_concurrency(0), - my_queue(q), forwarder_busy(false) { + function_input_base( + graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(size_t max_concurrency, node_priority_t priority) + ) : my_graph_ref(g), my_max_concurrency(max_concurrency) + , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(priority)) + , my_queue(!internal::has_policy::value ? new input_queue_type() : NULL) + , forwarder_busy(false) + { my_predecessors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); } //! Copy constructor - function_input_base( const function_input_base& src, input_queue_type *q = NULL) : - receiver(), tbb::internal::no_assign(), - my_graph_ptr(src.my_graph_ptr), my_max_concurrency(src.my_max_concurrency), - my_concurrency(0), my_queue(q), forwarder_busy(false) + function_input_base( const function_input_base& src) + : receiver(), tbb::internal::no_assign() + , my_graph_ref(src.my_graph_ref), my_max_concurrency(src.my_max_concurrency) + , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(src.my_priority)) + , my_queue(src.my_queue ? new input_queue_type() : NULL), forwarder_busy(false) { my_predecessors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); @@ -111,18 +114,8 @@ namespace internal { if ( my_queue ) delete my_queue; } - //! Put to the node, returning a task if available - task * try_put_task( const input_type &t ) __TBB_override { - if ( my_max_concurrency == 0 ) { - return create_body_task( t ); - } else { - operation_type op_data(t, tryput_bypass); - my_aggregator.execute(&op_data); - if(op_data.status == internal::SUCCEEDED) { - return op_data.bypass_t; - } - return NULL; - } + task* try_put_task( const input_type& t) __TBB_override { + return try_put_task_impl(t, internal::has_policy()); } //! Adds src to the list of cached predecessors. @@ -141,7 +134,7 @@ namespace internal { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! Adds to list of predecessors added by make_edge void internal_add_built_predecessor( predecessor_type &src) __TBB_override { operation_type op_data(add_blt_pred); @@ -171,7 +164,7 @@ namespace internal { built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: @@ -184,9 +177,10 @@ namespace internal { forwarder_busy = false; } - graph* my_graph_ptr; + graph& my_graph_ref; const size_t my_max_concurrency; size_t my_concurrency; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) input_queue_type *my_queue; predecessor_cache my_predecessors; @@ -197,6 +191,16 @@ namespace internal { __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); } + graph& graph_reference() __TBB_override { + return my_graph_ref; + } + + task* try_get_postponed_task(const input_type& i) { + operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + private: friend class apply_body_task_bypass< class_type, input_type >; @@ -208,10 +212,10 @@ namespace internal { union { input_type *elem; predecessor_type *r; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; predecessor_list_type *predv; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ }; tbb::task *bypass_t; operation_type(const input_type& e, op_type t) : @@ -224,7 +228,7 @@ namespace internal { friend class internal::aggregating_functor; aggregator< handler_type, operation_type > my_aggregator; - task* create_and_spawn_task(bool spawn) { + task* perform_queued_requests() { task* new_task = NULL; if(my_queue) { if(!my_queue->empty()) { @@ -241,13 +245,6 @@ namespace internal { new_task = create_body_task(i); } } - //! Spawns a task that applies a body - // task == NULL => g.reset(), which shouldn't occur in concurrent context - if(spawn && new_task) { - FLOW_SPAWN(*new_task); - new_task = SUCCESSFULLY_ENQUEUED; - } - return new_task; } void handle_operations(operation_type *op_list) { @@ -268,27 +265,27 @@ namespace internal { my_predecessors.remove(*(tmp->r)); __TBB_store_with_release(tmp->status, SUCCEEDED); break; - case app_body: - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (my_concurrencybypass_t = NULL; __TBB_ASSERT(my_max_concurrency != 0, NULL); --my_concurrency; if(my_concurrencybypass_t = create_and_spawn_task(/*spawn=*/false); + tmp->bypass_t = perform_queued_requests(); __TBB_store_with_release(tmp->status, SUCCEEDED); } break; case tryput_bypass: internal_try_put_task(tmp); break; case try_fwd: internal_forward(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + case occupy_concurrency: + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + __TBB_store_with_release(tmp->status, SUCCEEDED); + } else { + __TBB_store_with_release(tmp->status, FAILED); + } + break; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: { my_predecessors.internal_add_built_predecessor(*(tmp->r)); __TBB_store_with_release(tmp->status, SUCCEEDED); @@ -306,7 +303,7 @@ namespace internal { my_predecessors.copy_predecessors( *(tmp->predv) ); __TBB_store_with_release(tmp->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -328,11 +325,11 @@ namespace internal { } } - //! Tries to spawn bodies if available and if concurrency allows + //! Creates tasks for postponed messages if available and if concurrency allows void internal_forward(operation_type *op) { op->bypass_t = NULL; if (my_concurrency < my_max_concurrency || !my_max_concurrency) - op->bypass_t = create_and_spawn_task(/*spawn=*/false); + op->bypass_t = perform_queued_requests(); if(op->bypass_t) __TBB_store_with_release(op->status, SUCCEEDED); else { @@ -341,81 +338,108 @@ namespace internal { } } + task* internal_try_put_bypass( const input_type& t ) { + operation_type op_data(t, tryput_bypass); + my_aggregator.execute(&op_data); + if( op_data.status == internal::SUCCEEDED ) { + return op_data.bypass_t; + } + return NULL; + } + + task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::true_type ) { + if( my_max_concurrency == 0 ) { + return apply_body_bypass(t); + } else { + operation_type check_op(t, occupy_concurrency); + my_aggregator.execute(&check_op); + if( check_op.status == internal::SUCCEEDED ) { + return apply_body_bypass(t); + } + return internal_try_put_bypass(t); + } + } + + task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::false_type ) { + if( my_max_concurrency == 0 ) { + return create_body_task(t); + } else { + return internal_try_put_bypass(t); + } + } + //! Applies the body to the provided input // then decides if more work is available - task * apply_body_bypass( input_type &i ) { - task * new_task = static_cast(this)->apply_body_impl_bypass(i); - if ( my_max_concurrency != 0 ) { - operation_type op_data(app_body_bypass); // tries to pop an item or get_item, enqueues another apply_body - my_aggregator.execute(&op_data); - tbb::task *ttask = op_data.bypass_t; - new_task = combine_tasks(new_task, ttask); - } - return new_task; + task * apply_body_bypass( const input_type &i ) { + return static_cast(this)->apply_body_impl_bypass(i); } //! allocates a task to apply a body inline task * create_body_task( const input_type &input ) { - - return (my_graph_ptr->is_active()) ? - new(task::allocate_additional_child_of(*(my_graph_ptr->root_task()))) - apply_body_task_bypass < class_type, input_type >(*this, input) : - NULL; + return (internal::is_graph_active(my_graph_ref)) ? + new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) + apply_body_task_bypass < class_type, input_type >( + *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(input, my_priority)) + : NULL; } //! This is executed by an enqueued task, the "forwarder" - task *forward_task() { + task* forward_task() { operation_type op_data(try_fwd); - task *rval = NULL; + task* rval = NULL; do { op_data.status = WAIT; my_aggregator.execute(&op_data); if(op_data.status == SUCCEEDED) { - tbb::task *ttask = op_data.bypass_t; - rval = combine_tasks(rval, ttask); + task* ttask = op_data.bypass_t; + __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, NULL ); + rval = combine_tasks(my_graph_ref, rval, ttask); } } while (op_data.status == SUCCEEDED); return rval; } inline task *create_forward_task() { - return (my_graph_ptr->is_active()) ? - new(task::allocate_additional_child_of(*(my_graph_ptr->root_task()))) forward_task_bypass< class_type >(*this) : - NULL; + return (internal::is_graph_active(my_graph_ref)) ? + new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) + forward_task_bypass< class_type >( __TBB_FLOW_GRAPH_PRIORITY_ARG1(*this, my_priority) ) + : NULL; } //! Spawns a task that calls forward() inline void spawn_forward_task() { task* tp = create_forward_task(); if(tp) { - FLOW_SPAWN(*tp); + internal::spawn_in_graph_arena(graph_reference(), *tp); } } }; // function_input_base //! Implements methods for a function node that takes a type Input as input and sends // a type Output to its successors. - template< typename Input, typename Output, typename A> - class function_input : public function_input_base > { + template< typename Input, typename Output, typename Policy, typename A> + class function_input : public function_input_base > { public: typedef Input input_type; typedef Output output_type; typedef function_body function_body_type; - typedef function_input my_class; - typedef function_input_base base_type; + typedef function_input my_class; + typedef function_input_base base_type; typedef function_input_queue input_queue_type; // constructor template - function_input( graph &g, size_t max_concurrency, Body& body, input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { + function_input( + graph &g, size_t max_concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } //! Copy constructor - function_input( const function_input& src, input_queue_type *q = NULL ) : - base_type(src, q), + function_input( const function_input& src ) : + base_type(src), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ) { } @@ -431,18 +455,52 @@ namespace internal { return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); } - task * apply_body_impl_bypass( const input_type &i) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE + output_type apply_body_impl( const input_type& i) { // There is an extra copied needed to capture the // body execution without the try_put tbb::internal::fgt_begin_body( my_body ); output_type v = (*my_body)(i); tbb::internal::fgt_end_body( my_body ); - task * new_task = successors().try_put_task( v ); + return v; + } + + //TODO: consider moving into the base class + task * apply_body_impl_bypass( const input_type &i) { + output_type v = apply_body_impl(i); +#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER + task* successor_task = successors().try_put_task(v); +#endif + task* postponed_task = NULL; + if( base_type::my_max_concurrency != 0 ) { + postponed_task = base_type::try_get_postponed_task(i); + __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, NULL ); + } +#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER + graph& g = base_type::my_graph_ref; + return combine_tasks(g, successor_task, postponed_task); #else - task * new_task = successors().try_put_task( (*my_body)(i) ); + if( postponed_task ) { + // make the task available for other workers since we do not know successors' + // execution policy + internal::spawn_in_graph_arena(base_type::graph_reference(), *postponed_task); + } + task* successor_task = successors().try_put_task(v); +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ #endif - return new_task; + if(internal::has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + if(!successor_task) { + // Return confirmative status since current + // node's body has been executed anyway + successor_task = SUCCESSFULLY_ENQUEUED; + } + } + return successor_task; +#endif /* TBB_DEPRECATED_MESSAGE_FLOW_ORDER */ } protected: @@ -485,7 +543,7 @@ namespace internal { } }; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // helper templates to extract the output ports of an multifunction_node from graph template struct extract_element { template static void extract_this(P &p) { @@ -503,32 +561,29 @@ namespace internal { //! Implements methods for a function node that takes a type Input as input // and has a tuple of output ports specified. - template< typename Input, typename OutputPortSet, typename A> - class multifunction_input : public function_input_base > { + template< typename Input, typename OutputPortSet, typename Policy, typename A> + class multifunction_input : public function_input_base > { public: static const int N = tbb::flow::tuple_size::value; typedef Input input_type; typedef OutputPortSet output_ports_type; typedef multifunction_body multifunction_body_type; - typedef multifunction_input my_class; - typedef function_input_base base_type; + typedef multifunction_input my_class; + typedef function_input_base base_type; typedef function_input_queue input_queue_type; // constructor template - multifunction_input( - graph &g, - size_t max_concurrency, - Body& body, - input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::multifunction_body_leaf(body) ), - my_init_body( new internal::multifunction_body_leaf(body) ) { + multifunction_input(graph &g, size_t max_concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) + , my_body( new internal::multifunction_body_leaf(body) ) + , my_init_body( new internal::multifunction_body_leaf(body) ) { } //! Copy constructor - multifunction_input( const multifunction_input& src, input_queue_type *q = NULL ) : - base_type(src, q), + multifunction_input( const multifunction_input& src ) : + base_type(src), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ) { } @@ -546,18 +601,22 @@ namespace internal { // for multifunction nodes we do not have a single successor as such. So we just tell // the task we were successful. + //TODO: consider moving common parts with implementation in function_input into separate function task * apply_body_impl_bypass( const input_type &i) { tbb::internal::fgt_begin_body( my_body ); (*my_body)(i, my_output_ports); tbb::internal::fgt_end_body( my_body ); - task * new_task = SUCCESSFULLY_ENQUEUED; - return new_task; + task* ttask = NULL; + if(base_type::my_max_concurrency != 0) { + ttask = base_type::try_get_postponed_task(i); + } + return ttask ? ttask : SUCCESSFULLY_ENQUEUED; } output_ports_type &output_ports(){ return my_output_ports; } protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { extract_element::extract_this(my_output_ports); } @@ -586,26 +645,36 @@ namespace internal { return tbb::flow::get(op.output_ports()); } -// helper structs for split_node + inline void check_task_and_spawn(graph& g, task* t) { + if (t && t != SUCCESSFULLY_ENQUEUED) { + internal::spawn_in_graph_arena(g, *t); + } + } + + // helper structs for split_node template struct emit_element { template - static void emit_this(const T &t, P &p) { - (void)tbb::flow::get(p).try_put(tbb::flow::get(t)); - emit_element::emit_this(t,p); + static task* emit_this(graph& g, const T &t, P &p) { + // TODO: consider to collect all the tasks in task_list and spawn them all at once + task* last_task = tbb::flow::get(p).try_put_task(tbb::flow::get(t)); + check_task_and_spawn(g, last_task); + return emit_element::emit_this(g,t,p); } }; template<> struct emit_element<1> { template - static void emit_this(const T &t, P &p) { - (void)tbb::flow::get<0>(p).try_put(tbb::flow::get<0>(t)); + static task* emit_this(graph& g, const T &t, P &p) { + task* last_task = tbb::flow::get<0>(p).try_put_task(tbb::flow::get<0>(t)); + check_task_and_spawn(g, last_task); + return SUCCESSFULLY_ENQUEUED; } }; //! Implements methods for an executable node that takes continue_msg as input - template< typename Output > + template< typename Output, typename Policy> class continue_input : public continue_receiver { public: @@ -615,22 +684,27 @@ namespace internal { //! The output type of this receiver typedef Output output_type; typedef function_body function_body_type; + typedef continue_input class_type; template< typename Body > - continue_input( graph &g, Body& body ) - : my_graph_ptr(&g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } + continue_input( graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) ) + : continue_receiver(__TBB_FLOW_GRAPH_PRIORITY_ARG1(/*number_of_predecessors=*/0, priority)) + , my_graph_ref(g) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + { } template< typename Body > - continue_input( graph &g, int number_of_predecessors, Body& body ) - : continue_receiver( number_of_predecessors ), my_graph_ptr(&g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + continue_input( graph &g, int number_of_predecessors, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : continue_receiver( __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, priority) ) + , my_graph_ref(g) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } continue_input( const continue_input& src ) : continue_receiver(src), - my_graph_ptr(src.my_graph_ptr), + my_graph_ref(src.my_graph_ref), my_body( src.my_init_body->clone() ), my_init_body( src.my_init_body->clone() ) {} @@ -656,36 +730,48 @@ namespace internal { protected: - graph* my_graph_ptr; + graph& my_graph_ref; function_body_type *my_body; function_body_type *my_init_body; virtual broadcast_cache &successors() = 0; - friend class apply_body_task_bypass< continue_input< Output >, continue_msg >; + friend class apply_body_task_bypass< class_type, continue_msg >; //! Applies the body to the provided input task *apply_body_bypass( input_type ) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE // There is an extra copied needed to capture the // body execution without the try_put tbb::internal::fgt_begin_body( my_body ); output_type v = (*my_body)( continue_msg() ); tbb::internal::fgt_end_body( my_body ); return successors().try_put_task( v ); -#else - return successors().try_put_task( (*my_body)( continue_msg() ) ); -#endif } - //! Spawns a task that applies the body - task *execute( ) __TBB_override { - return (my_graph_ptr->is_active()) ? - new ( task::allocate_additional_child_of( *(my_graph_ptr->root_task()) ) ) - apply_body_task_bypass< continue_input< Output >, continue_msg >( *this, continue_msg() ) : - NULL; + task* execute() __TBB_override { + if(!internal::is_graph_active(my_graph_ref)) { + return NULL; + } +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(internal::has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + return apply_body_bypass( continue_msg() ); + } + else { + return new ( task::allocate_additional_child_of( *(my_graph_ref.root_task()) ) ) + apply_body_task_bypass< class_type, continue_msg >( + *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(continue_msg(), my_priority) ); + } } + graph& graph_reference() __TBB_override { + return my_graph_ref; + } }; // continue_input //! Implements methods for both executable and function nodes that puts Output to its successors @@ -697,7 +783,7 @@ namespace internal { typedef Output output_type; typedef typename sender::successor_type successor_type; typedef broadcast_cache broadcast_cache_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -719,7 +805,7 @@ namespace internal { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return successors().built_successors(); } @@ -738,7 +824,7 @@ namespace internal { void copy_successors( successor_list_type &v) __TBB_override { successors().copy_successors(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ // for multifunction_node. The function_body that implements // the node will have an input and an output tuple of ports. To put @@ -769,15 +855,26 @@ namespace internal { multifunction_output( const multifunction_output &/*other*/) : base_type() { my_successors.set_owner(this); } bool try_put(const output_type &i) { - task *res = my_successors.try_put_task(i); + task *res = try_put_task(i); if(!res) return false; - if(res != SUCCESSFULLY_ENQUEUED) FLOW_SPAWN(*res); + if(res != SUCCESSFULLY_ENQUEUED) { + FLOW_SPAWN(*res); // TODO: Spawn task inside arena + } return true; } + + protected: + + task* try_put_task(const output_type &i) { + return my_successors.try_put_task(i); + } + + template friend struct emit_element; + }; // multifunction_output //composite_node -#if TBB_PREVIEW_FLOW_GRAPH_TRACE && __TBB_FLOW_GRAPH_CPP11_FEATURES +#if __TBB_FLOW_GRAPH_CPP11_FEATURES template void add_nodes_impl(CompositeType*, bool) {} @@ -785,10 +882,7 @@ namespace internal { void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { void *addr = const_cast(&n1); - if(visible) - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, c_node, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_parent_of, addr, tbb::internal::FLOW_NODE ); - else - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, addr, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_child_of, c_node, tbb::internal::FLOW_NODE ); + fgt_alias_port(c_node, addr, visible); add_nodes_impl(c_node, visible, n...); } #endif diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_streaming_node.h b/inst/include/tbb_local/tbb/internal/_flow_graph_streaming_node.h index 493f76e8..2a1d43e0 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_streaming_node.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_streaming_node.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_streaming_H diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_tagged_buffer_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_tagged_buffer_impl.h index 46755fe0..87a32fe6 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_tagged_buffer_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_tagged_buffer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // a hash table buffer that can expand, and can support as many deletions as diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_trace_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_trace_impl.h index b798888d..aee40762 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_trace_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_trace_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _FGT_GRAPH_TRACE_IMPL_H @@ -26,7 +22,67 @@ namespace tbb { namespace internal { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE +#if TBB_USE_THREADING_TOOLS + +static inline void fgt_alias_port(void *node, void *p, bool visible) { + if(visible) + itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE ); + else + itt_relation_add( ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +static inline void fgt_composite ( void *node, void *graph ) { + itt_make_task_group( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE ); +} + +static inline void fgt_internal_alias_input_port( void *node, void *p, string_index name_index ) { + itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT ); +} + +static inline void fgt_internal_alias_output_port( void *node, void *p, string_index name_index ) { + itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT ); +} + +template +void alias_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { + // TODO: Make fgt_internal_alias_input_port a function template? + fgt_internal_alias_input_port( node, port, name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); + fgt_internal_input_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { } +}; + +template +void alias_output_port(void *node, tbb::flow::sender* port, string_index name_index) { + // TODO: Make fgt_internal_alias_output_port a function template? + fgt_internal_alias_output_port( node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_output_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); + fgt_internal_output_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) { + } +}; static inline void fgt_internal_create_input_port( void *node, void *p, string_index name_index ) { itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); @@ -38,8 +94,11 @@ static inline void fgt_internal_create_output_port( void *node, void *p, string_ template void register_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - //TODO: Make fgt_internal_create_input_port a function template? - fgt_internal_create_input_port( node, port, name_index); + // TODO: Make fgt_internal_create_input_port a function template? + // In C++03 dependent name lookup from the template definition context + // works only for function declarations with external linkage: + // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#561 + fgt_internal_create_input_port(node, static_cast(port), name_index); } template < typename PortsTuple, int N > @@ -59,7 +118,7 @@ struct fgt_internal_input_helper { template void register_output_port(void *node, tbb::flow::sender* port, string_index name_index) { - //TODO: Make fgt_internal_create_output_port a function template? + // TODO: Make fgt_internal_create_output_port a function template? fgt_internal_create_output_port( node, static_cast(port), name_index); } @@ -126,6 +185,10 @@ static inline void fgt_multiinput_node( string_index t, void *g, PortsTuple &por fgt_internal_input_helper::register_port( output_port, ports ); } +static inline void fgt_multiinput_multioutput_node( string_index t, void *n, void *g ) { + itt_make_task_group( ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t ); +} + static inline void fgt_node( string_index t, void *g, void *output_port ) { itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); fgt_internal_create_output_port( output_port, output_port, FLOW_OUTPUT_PORT_0 ); @@ -186,11 +249,23 @@ static inline void fgt_async_reserve( void *node, void *graph ) { itt_region_begin( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); } -static inline void fgt_async_commit( void *node, void *graph ) { +static inline void fgt_async_commit( void *node, void * /*graph*/) { itt_region_end( ITT_DOMAIN_FLOW, node, FLOW_NODE ); } -#else // TBB_PREVIEW_FLOW_GRAPH_TRACE +static inline void fgt_reserve_wait( void *graph ) { + itt_region_begin( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_NULL ); +} + +static inline void fgt_release_wait( void *graph ) { + itt_region_end( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); +} + +#else // TBB_USE_THREADING_TOOLS + +static inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { } + +static inline void fgt_composite ( void * /*node*/, void * /*graph*/ ) { } static inline void fgt_graph( void * /*g*/ ) { } @@ -213,6 +288,8 @@ static inline void fgt_multioutput_node_with_body( string_index /*t*/, void * /* template< int N, typename PortsTuple > static inline void fgt_multiinput_node( string_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } +static inline void fgt_multiinput_multioutput_node( string_index /*t*/, void * /*node*/, void * /*graph*/ ) { } + static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*output_port*/ ) { } static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } static inline void fgt_node( string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } @@ -225,12 +302,28 @@ static inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port* static inline void fgt_begin_body( void * /*body*/ ) { } static inline void fgt_end_body( void * /*body*/) { } + static inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } static inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } static inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } +static inline void fgt_reserve_wait( void * /*graph*/ ) { } +static inline void fgt_release_wait( void * /*graph*/ ) { } + +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; -#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE +#endif // TBB_USE_THREADING_TOOLS } // namespace internal } // namespace tbb diff --git a/inst/include/tbb_local/tbb/internal/_flow_graph_types_impl.h b/inst/include/tbb_local/tbb/internal/_flow_graph_types_impl.h index 73b5f547..e223dae6 100644 --- a/inst/include/tbb_local/tbb/internal/_flow_graph_types_impl.h +++ b/inst/include/tbb_local/tbb/internal/_flow_graph_types_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_types_impl_H @@ -48,7 +44,7 @@ namespace internal { typedef KHashp KHash; }; -// wrap each element of a tuple in a template, and make a tuple of the result. + // wrap each element of a tuple in a template, and make a tuple of the result. template class PT, typename TypeTuple> struct wrap_tuple_elements; @@ -57,6 +53,19 @@ namespace internal { template class PT, typename KeyTraits, typename TypeTuple> struct wrap_key_tuple_elements; +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT + template class PT, typename... Args> + struct wrap_tuple_elements >{ + typedef typename tbb::flow::tuple... > type; + }; + + template class PT, typename KeyTraits, typename... Args> + struct wrap_key_tuple_elements > { + typedef typename KeyTraits::key_type K; + typedef typename KeyTraits::hash_compare_type KHash; + typedef typename tbb::flow::tuple >... > type; + }; +#else template class PT, typename TypeTuple> struct wrap_tuple_elements<1, PT, TypeTuple> { typedef typename tbb::flow::tuple< @@ -314,6 +323,7 @@ namespace internal { PT > type; }; #endif +#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT */ #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT template< int... S > class sequence {}; diff --git a/inst/include/tbb_local/tbb/internal/_mutex_padding.h b/inst/include/tbb_local/tbb/internal/_mutex_padding.h index 6c1d9b59..09fccd44 100644 --- a/inst/include/tbb_local/tbb/internal/_mutex_padding.h +++ b/inst/include/tbb_local/tbb/internal/_mutex_padding.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mutex_padding_H diff --git a/inst/include/tbb_local/tbb/internal/_node_handle_impl.h b/inst/include/tbb_local/tbb/internal/_node_handle_impl.h new file mode 100644 index 00000000..a910b5fa --- /dev/null +++ b/inst/include/tbb_local/tbb/internal/_node_handle_impl.h @@ -0,0 +1,168 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_node_handle_H +#define __TBB_node_handle_H + +#include "_allocator_traits.h" +#include "../tbb_config.h" + + +namespace tbb { + +// This classes must be declared here for correct friendly relationship +// TODO: Consider creation some internal class to access node_handle private fields without any friendly classes +namespace interface5 { +namespace internal { + template + class split_ordered_list; + template + class concurrent_unordered_base; +} +} + +namespace interface10{ +namespace internal { + template + class concurrent_skip_list; +} +} + +namespace internal { + +template +class node_handle_base { +public: + typedef Allocator allocator_type; +protected: + typedef Node node; + typedef tbb::internal::allocator_traits traits_type; +public: + + node_handle_base() : my_node(NULL), my_allocator() {} + node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), + my_allocator(std::move(nh.my_allocator)) { + nh.my_node = NULL; + } + + bool empty() const { return my_node == NULL; } + explicit operator bool() const { return my_node != NULL; } + + ~node_handle_base() { internal_destroy(); } + + node_handle_base& operator=(node_handle_base&& nh) { + internal_destroy(); + my_node = nh.my_node; + typedef typename traits_type::propagate_on_container_move_assignment pocma_type; + tbb::internal::allocator_move_assignment(my_allocator, nh.my_allocator, pocma_type()); + nh.deactivate(); + return *this; + } + + void swap(node_handle_base& nh) { + std::swap(my_node, nh.my_node); + typedef typename traits_type::propagate_on_container_swap pocs_type; + tbb::internal::allocator_swap(my_allocator, nh.my_allocator, pocs_type()); + } + + allocator_type get_allocator() const { + return my_allocator; + } + +protected: + node_handle_base(node* n) : my_node(n) {} + + void internal_destroy() { + if(my_node) { + traits_type::destroy(my_allocator, my_node->storage()); + typename tbb::internal::allocator_rebind::type node_allocator; + node_allocator.deallocate(my_node, 1); + } + } + + void deactivate() { my_node = NULL; } + + node* my_node; + allocator_type my_allocator; +}; + +// node handle for maps +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key key_type; + typedef typename Value::second_type mapped_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + key_type& key() const { + __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); + return *const_cast(&(this->my_node->value().first)); + } + + mapped_type& mapped() const { + __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); + return this->my_node->value().second; + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + +// node handle for sets +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key value_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + value_type& value() const { + __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); + return *const_cast(&(this->my_node->value())); + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + + +}// namespace internal +}// namespace tbb + +#endif /*__TBB_node_handle_H*/ diff --git a/inst/include/tbb_local/tbb/internal/_range_iterator.h b/inst/include/tbb_local/tbb/internal/_range_iterator.h index 5ebc42e8..733c795f 100644 --- a/inst/include/tbb_local/tbb/internal/_range_iterator.h +++ b/inst/include/tbb_local/tbb/internal/_range_iterator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_range_iterator_H diff --git a/inst/include/tbb_local/tbb/internal/_tbb_hash_compare_impl.h b/inst/include/tbb_local/tbb/internal/_tbb_hash_compare_impl.h index 6381e2dc..510bde32 100644 --- a/inst/include/tbb_local/tbb/internal/_tbb_hash_compare_impl.h +++ b/inst/include/tbb_local/tbb/internal/_tbb_hash_compare_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // must be included outside namespaces. diff --git a/inst/include/tbb_local/tbb/internal/_tbb_strings.h b/inst/include/tbb_local/tbb/internal/_tbb_strings.h index 1aa532f3..a5fd3ce5 100644 --- a/inst/include/tbb_local/tbb/internal/_tbb_strings.h +++ b/inst/include/tbb_local/tbb/internal/_tbb_strings.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ TBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, "broadcast_node") @@ -65,3 +61,16 @@ TBB_STRING_RESOURCE(FLOW_NULL, "null") TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") +TBB_STRING_RESOURCE(FLOW_OPENCL_NODE, "opencl_node") +TBB_STRING_RESOURCE(ALGORITHM, "tbb_algorithm") +TBB_STRING_RESOURCE(PARALLEL_FOR, "tbb_parallel_for") +TBB_STRING_RESOURCE(PARALLEL_DO, "tbb_parallel_do") +TBB_STRING_RESOURCE(PARALLEL_INVOKE, "tbb_parallel_invoke") +TBB_STRING_RESOURCE(PARALLEL_REDUCE, "tbb_parallel_reduce") +TBB_STRING_RESOURCE(PARALLEL_SCAN, "tbb_parallel_scan") +TBB_STRING_RESOURCE(PARALLEL_SORT, "tbb_parallel_sort") +TBB_STRING_RESOURCE(CUSTOM_CTX, "tbb_custom") +TBB_STRING_RESOURCE(FLOW_TASKS, "tbb_flow_graph") +TBB_STRING_RESOURCE(PARALLEL_FOR_TASK, "tbb_parallel_for_task") +// TODO: Drop following string prefix "fgt_" here and in FGA's collector +TBB_STRING_RESOURCE(USER_EVENT, "fgt_user_event") diff --git a/inst/include/tbb_local/tbb/internal/_tbb_trace_impl.h b/inst/include/tbb_local/tbb/internal/_tbb_trace_impl.h index e7eb5784..e89ab232 100644 --- a/inst/include/tbb_local/tbb/internal/_tbb_trace_impl.h +++ b/inst/include/tbb_local/tbb/internal/_tbb_trace_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _FGT_TBB_TRACE_IMPL_H @@ -27,18 +23,17 @@ namespace tbb { namespace internal { #if TBB_PREVIEW_ALGORITHM_TRACE - static inline void fgt_algorithm( string_index t, void *algorithm, void *parent ) { - itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, parent, FGT_ALGORITHM, t ); + itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, parent, ALGORITHM, t ); } static inline void fgt_begin_algorithm( string_index t, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, NULL, FLOW_NULL, t ); + itt_task_begin( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, NULL, FLOW_NULL, t ); } static inline void fgt_end_algorithm( void * ) { itt_task_end( ITT_DOMAIN_FLOW ); } static inline void fgt_alg_begin_body( string_index t, void *body, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, FGT_ALGORITHM, t ); + itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, ALGORITHM, t ); } static inline void fgt_alg_end_body( void * ) { itt_task_end( ITT_DOMAIN_FLOW ); diff --git a/inst/include/tbb_local/tbb/internal/_tbb_windef.h b/inst/include/tbb_local/tbb/internal/_tbb_windef.h index e798dee4..1268ba27 100644 --- a/inst/include/tbb_local/tbb/internal/_tbb_windef.h +++ b/inst/include/tbb_local/tbb/internal/_tbb_windef.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,17 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_windef_H #error Do not #include this internal file directly; use public TBB headers instead. #endif /* __TBB_tbb_windef_H */ -// Check that the target Windows version has all API calls requried for TBB. +// Check that the target Windows version has all API calls required for TBB. // Do not increase the version in condition beyond 0x0500 without prior discussion! #if defined(_WIN32_WINNT) && _WIN32_WINNT<0x0501 #error TBB is unable to run on old Windows versions; _WIN32_WINNT must be 0x0501 or greater. diff --git a/inst/include/tbb_local/tbb/internal/_template_helpers.h b/inst/include/tbb_local/tbb/internal/_template_helpers.h index 1e0abbe8..87c3efbe 100644 --- a/inst/include/tbb_local/tbb/internal/_template_helpers.h +++ b/inst/include/tbb_local/tbb/internal/_template_helpers.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_template_helpers_H @@ -23,6 +19,14 @@ #include #include +#include "../tbb_config.h" +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT +#include +#endif +#if __TBB_CPP11_PRESENT +#include +#include // allocator_traits +#endif namespace tbb { namespace internal { @@ -60,11 +64,41 @@ template struct is_same_type { static const bool value = template struct is_ref { static const bool value = false; }; template struct is_ref { static const bool value = true; }; +//! Partial support for std::is_integral +template struct is_integral_impl { static const bool value = false; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#if __TBB_CPP11_PRESENT +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#endif +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; + +template +struct is_integral : is_integral_impl::type> {}; + #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT //! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) template struct void_t { typedef void type; }; #endif +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502 +template class... Checks> +struct supports_impl { typedef std::false_type type; }; +template class... Checks> +struct supports_impl...>::type, Checks...> { typedef std::true_type type; }; + +template class... Checks> +using supports = typename supports_impl::type; + +#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ + #if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT //! Allows to store a function parameter pack as a variable and later pass it to another function @@ -153,6 +187,81 @@ stored_pack save_pack( Types&&... types ) { } #endif /* __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ + +#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT + +using std::index_sequence; +using std::make_index_sequence; + +#elif __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +template class index_sequence {}; + +template +struct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {}; + +template +struct make_index_sequence_impl <0, S...> { + using type = index_sequence; +}; + +template +using make_index_sequence = typename tbb::internal::make_index_sequence_impl::type; + +#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */ + +#if __TBB_CPP11_PRESENT + +template< typename Iter > +using iterator_value_t = typename std::iterator_traits::value_type; + +template< typename Iter > +using iterator_key_t = typename std::remove_const::first_type>::type; + +template< typename Iter > +using iterator_mapped_t = typename iterator_value_t::second_type; + +template< typename A > using value_type = typename A::value_type; +template< typename A > using alloc_ptr_t = typename std::allocator_traits::pointer; +template< typename A > using has_allocate = decltype(std::declval&>() = std::declval().allocate(0)); +template< typename A > using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0)); + +// value_type should be checked first because it can be used in other checks (via allocator_traits) +template< typename T > +using is_allocator = supports; + +#if __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT + +template< typename T > +static constexpr bool is_allocator_v = is_allocator::value; + +#endif /*__TBB_CPP14_VARIABLE_TEMPLATES */ + +template< std::size_t N, typename... Args > +struct pack_element { + using type = void; +}; + +template< std::size_t N, typename T, typename... Args > +struct pack_element { + using type = typename pack_element::type; +}; + +template< typename T, typename... Args > +struct pack_element<0, T, Args...> { + using type = T; +}; + +template< std::size_t N, typename... Args > +using pack_element_t = typename pack_element::type; + +template using is_transparent = typename Comp::is_transparent; + +template +using has_is_transparent = supports; + +#endif /* __TBB_CPP11_PRESENT */ + } } // namespace internal, namespace tbb #endif /* __TBB_template_helpers_H */ diff --git a/inst/include/tbb_local/tbb/internal/_x86_eliding_mutex_impl.h b/inst/include/tbb_local/tbb/internal/_x86_eliding_mutex_impl.h index ef5f9223..a03e463f 100644 --- a/inst/include/tbb_local/tbb/internal/_x86_eliding_mutex_impl.h +++ b/inst/include/tbb_local/tbb/internal/_x86_eliding_mutex_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__x86_eliding_mutex_impl_H diff --git a/inst/include/tbb_local/tbb/internal/_x86_rtm_rw_mutex_impl.h b/inst/include/tbb_local/tbb/internal/_x86_rtm_rw_mutex_impl.h index b08c2331..0003abac 100644 --- a/inst/include/tbb_local/tbb/internal/_x86_rtm_rw_mutex_impl.h +++ b/inst/include/tbb_local/tbb/internal/_x86_rtm_rw_mutex_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__x86_rtm_rw_mutex_impl_H @@ -88,11 +84,11 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { static x86_rtm_rw_mutex* internal_get_mutex( const spin_rw_mutex::scoped_lock& lock ) { - return static_cast( lock.internal_get_mutex() ); + return static_cast( lock.mutex ); } static void internal_set_mutex( spin_rw_mutex::scoped_lock& lock, spin_rw_mutex* mtx ) { - lock.internal_set_mutex( mtx ); + lock.mutex = mtx; } //! @endcond public: @@ -171,7 +167,8 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { bool upgrade_to_writer() { x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_reader || transaction_state==RTM_real_reader, "Invalid state for upgrade" ); + if (transaction_state == RTM_transacting_writer || transaction_state == RTM_real_writer) + return true; // Already a writer return mutex->internal_upgrade(*this); } @@ -180,7 +177,8 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { bool downgrade_to_reader() { x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_writer || transaction_state==RTM_real_writer, "Invalid state for downgrade" ); + if (transaction_state == RTM_transacting_reader || transaction_state == RTM_real_reader) + return true; // Already a reader return mutex->internal_downgrade(*this); } diff --git a/inst/include/tbb_local/tbb/iterators.h b/inst/include/tbb_local/tbb/iterators.h new file mode 100644 index 00000000..7a3c9273 --- /dev/null +++ b/inst/include/tbb_local/tbb/iterators.h @@ -0,0 +1,326 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_iterators_H +#define __TBB_iterators_H + +#include +#include + +#include "tbb_config.h" +#include "tbb_stddef.h" + +#if __TBB_CPP11_PRESENT + +#include + +namespace tbb { + +template +class counting_iterator { + __TBB_STATIC_ASSERT(std::numeric_limits::is_integer, "Cannot instantiate counting_iterator with a non-integer type"); +public: + typedef typename std::make_signed::type difference_type; + typedef IntType value_type; + typedef const IntType* pointer; + typedef const IntType& reference; + typedef std::random_access_iterator_tag iterator_category; + + counting_iterator() : my_counter() {} + explicit counting_iterator(IntType init) : my_counter(init) {} + + reference operator*() const { return my_counter; } + value_type operator[](difference_type i) const { return *(*this + i); } + + difference_type operator-(const counting_iterator& it) const { return my_counter - it.my_counter; } + + counting_iterator& operator+=(difference_type forward) { my_counter += forward; return *this; } + counting_iterator& operator-=(difference_type backward) { return *this += -backward; } + counting_iterator& operator++() { return *this += 1; } + counting_iterator& operator--() { return *this -= 1; } + + counting_iterator operator++(int) { + counting_iterator it(*this); + ++(*this); + return it; + } + counting_iterator operator--(int) { + counting_iterator it(*this); + --(*this); + return it; + } + + counting_iterator operator-(difference_type backward) const { return counting_iterator(my_counter - backward); } + counting_iterator operator+(difference_type forward) const { return counting_iterator(my_counter + forward); } + friend counting_iterator operator+(difference_type forward, const counting_iterator it) { return it + forward; } + + bool operator==(const counting_iterator& it) const { return *this - it == 0; } + bool operator!=(const counting_iterator& it) const { return !(*this == it); } + bool operator<(const counting_iterator& it) const {return *this - it < 0; } + bool operator>(const counting_iterator& it) const { return it < *this; } + bool operator<=(const counting_iterator& it) const { return !(*this > it); } + bool operator>=(const counting_iterator& it) const { return !(*this < it); } + +private: + IntType my_counter; +}; +} //namespace tbb + + +#include + +#include "internal/_template_helpers.h" // index_sequence, make_index_sequence + +namespace tbb { +namespace internal { + +template +struct tuple_util { + template + static void increment(TupleType& it, DifferenceType forward) { + std::get(it) += forward; + tuple_util::increment(it, forward); + } + template + static bool check_sync(const TupleType& it1, const TupleType& it2, DifferenceType val) { + if(std::get(it1) - std::get(it2) != val) + return false; + return tuple_util::check_sync(it1, it2, val); + } +}; + +template<> +struct tuple_util<0> { + template + static void increment(TupleType&, DifferenceType) {} + template + static bool check_sync(const TupleType&, const TupleType&, DifferenceType) { return true;} +}; + +template +struct make_references { + template + TupleReturnType operator()(const TupleType& t, tbb::internal::index_sequence) { + return std::tie( *std::get(t)... ); + } +}; + +// A simple wrapper over a tuple of references. +// The class is designed to hold a temporary tuple of reference +// after dereferencing a zip_iterator; in particular, it is needed +// to swap these rvalue tuples. Any other usage is not supported. +template +struct tuplewrapper : public std::tuple::value, T&&>::type...> { + // In the context of this class, T is a reference, so T&& is a "forwarding reference" + typedef std::tuple base_type; + // Construct from the result of std::tie + tuplewrapper(const base_type& in) : base_type(in) {} +#if __INTEL_COMPILER + // ICC cannot generate copy ctor & assignment + tuplewrapper(const tuplewrapper& rhs) : base_type(rhs) {} + tuplewrapper& operator=(const tuplewrapper& rhs) { + *this = base_type(rhs); + return *this; + } +#endif + // Assign any tuple convertible to std::tuple: *it = a_tuple; + template + tuplewrapper& operator=(const std::tuple& other) { + base_type::operator=(other); + return *this; + } +#if _LIBCPP_VERSION + // (Necessary for libc++ tuples) Convert to a tuple of values: v = *it; + operator std::tuple::type...>() { return base_type(*this); } +#endif + // Swap rvalue tuples: swap(*it1,*it2); + friend void swap(tuplewrapper&& a, tuplewrapper&& b) { + std::swap(a,b); + } +}; + +} //namespace internal + +template +class zip_iterator { + __TBB_STATIC_ASSERT(sizeof...(Types), "Cannot instantiate zip_iterator with empty template parameter pack"); + static const std::size_t num_types = sizeof...(Types); + typedef std::tuple it_types; +public: + typedef typename std::make_signed::type difference_type; + typedef std::tuple::value_type...> value_type; +#if __INTEL_COMPILER && __INTEL_COMPILER < 1800 && _MSC_VER + typedef std::tuple::reference...> reference; +#else + typedef tbb::internal::tuplewrapper::reference...> reference; +#endif + typedef std::tuple::pointer...> pointer; + typedef std::random_access_iterator_tag iterator_category; + + zip_iterator() : my_it() {} + explicit zip_iterator(Types... args) : my_it(std::make_tuple(args...)) {} + zip_iterator(const zip_iterator& input) : my_it(input.my_it) {} + zip_iterator& operator=(const zip_iterator& input) { + my_it = input.my_it; + return *this; + } + + reference operator*() const { + return tbb::internal::make_references()(my_it, tbb::internal::make_index_sequence()); + } + reference operator[](difference_type i) const { return *(*this + i); } + + difference_type operator-(const zip_iterator& it) const { + __TBB_ASSERT(internal::tuple_util::check_sync(my_it, it.my_it, std::get<0>(my_it) - std::get<0>(it.my_it)), + "Components of zip_iterator are not synchronous"); + return std::get<0>(my_it) - std::get<0>(it.my_it); + } + + zip_iterator& operator+=(difference_type forward) { + internal::tuple_util::increment(my_it, forward); + return *this; + } + zip_iterator& operator-=(difference_type backward) { return *this += -backward; } + zip_iterator& operator++() { return *this += 1; } + zip_iterator& operator--() { return *this -= 1; } + + zip_iterator operator++(int) { + zip_iterator it(*this); + ++(*this); + return it; + } + zip_iterator operator--(int) { + zip_iterator it(*this); + --(*this); + return it; + } + + zip_iterator operator-(difference_type backward) const { + zip_iterator it(*this); + return it -= backward; + } + zip_iterator operator+(difference_type forward) const { + zip_iterator it(*this); + return it += forward; + } + friend zip_iterator operator+(difference_type forward, const zip_iterator& it) { return it + forward; } + + bool operator==(const zip_iterator& it) const { + return *this - it == 0; + } + it_types base() const { return my_it; } + + bool operator!=(const zip_iterator& it) const { return !(*this == it); } + bool operator<(const zip_iterator& it) const { return *this - it < 0; } + bool operator>(const zip_iterator& it) const { return it < *this; } + bool operator<=(const zip_iterator& it) const { return !(*this > it); } + bool operator>=(const zip_iterator& it) const { return !(*this < it); } +private: + it_types my_it; +}; + +template +zip_iterator make_zip_iterator(T... args) { return zip_iterator(args...); } + +template +class transform_iterator { +public: + typedef typename std::iterator_traits::value_type value_type; + typedef typename std::iterator_traits::difference_type difference_type; +#if __TBB_CPP17_INVOKE_RESULT_PRESENT + typedef typename std::invoke_result::reference>::type reference; +#else + typedef typename std::result_of::reference)>::type reference; +#endif + typedef typename std::iterator_traits::pointer pointer; + typedef typename std::random_access_iterator_tag iterator_category; + + transform_iterator(Iter it, UnaryFunc unary_func) : my_it(it), my_unary_func(unary_func) { + __TBB_STATIC_ASSERT((std::is_same::iterator_category, + std::random_access_iterator_tag>::value), "Random access iterator required."); + } + transform_iterator(const transform_iterator& input) : my_it(input.my_it), my_unary_func(input.my_unary_func) { } + transform_iterator& operator=(const transform_iterator& input) { + my_it = input.my_it; + return *this; + } + reference operator*() const { + return my_unary_func(*my_it); + } + reference operator[](difference_type i) const { + return *(*this + i); + } + transform_iterator& operator++() { + ++my_it; + return *this; + } + transform_iterator& operator--() { + --my_it; + return *this; + } + transform_iterator operator++(int) { + transform_iterator it(*this); + ++(*this); + return it; + } + transform_iterator operator--(int) { + transform_iterator it(*this); + --(*this); + return it; + } + transform_iterator operator+(difference_type forward) const { + return { my_it + forward, my_unary_func }; + } + transform_iterator operator-(difference_type backward) const { + return { my_it - backward, my_unary_func }; + } + transform_iterator& operator+=(difference_type forward) { + my_it += forward; + return *this; + } + transform_iterator& operator-=(difference_type backward) { + my_it -= backward; + return *this; + } + friend transform_iterator operator+(difference_type forward, const transform_iterator& it) { + return it + forward; + } + difference_type operator-(const transform_iterator& it) const { + return my_it - it.my_it; + } + bool operator==(const transform_iterator& it) const { return *this - it == 0; } + bool operator!=(const transform_iterator& it) const { return !(*this == it); } + bool operator<(const transform_iterator& it) const { return *this - it < 0; } + bool operator>(const transform_iterator& it) const { return it < *this; } + bool operator<=(const transform_iterator& it) const { return !(*this > it); } + bool operator>=(const transform_iterator& it) const { return !(*this < it); } + + Iter base() const { return my_it; } +private: + Iter my_it; + const UnaryFunc my_unary_func; +}; + +template +transform_iterator make_transform_iterator(Iter it, UnaryFunc unary_func) { + return transform_iterator(it, unary_func); +} + +} //namespace tbb + +#endif //__TBB_CPP11_PRESENT + +#endif /* __TBB_iterators_H */ diff --git a/inst/include/tbb_local/tbb/machine/gcc_arm.h b/inst/include/tbb_local/tbb/machine/gcc_arm.h new file mode 100644 index 00000000..40118e87 --- /dev/null +++ b/inst/include/tbb_local/tbb/machine/gcc_arm.h @@ -0,0 +1,216 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Platform isolation layer for the ARMv7-a architecture. +*/ + +#ifndef __TBB_machine_H +#error Do not include this file directly; include tbb_machine.h instead +#endif + +#if __ARM_ARCH_7A__ + +#include +#include + +#define __TBB_WORDSIZE 4 + +// Traditionally ARM is little-endian. +// Note that, since only the layout of aligned 32-bit words is of interest, +// any apparent PDP-endianness of 32-bit words at half-word alignment or +// any little-endian ordering of big-endian 32-bit words in 64-bit quantities +// may be disregarded for this setting. +#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG +#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE +#elif defined(__BYTE_ORDER__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED +#else + #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT +#endif + + +#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") +#define __TBB_full_memory_fence() __asm__ __volatile__("dmb ish": : :"memory") +#define __TBB_control_consistency_helper() __TBB_full_memory_fence() +#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() +#define __TBB_release_consistency_helper() __TBB_full_memory_fence() + +//-------------------------------------------------- +// Compare and swap +//-------------------------------------------------- + +/** + * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr + * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand + * @param value value to assign *ptr to if *ptr==comparand + * @param comparand value to compare with *ptr + * @return value originally in memory at ptr, regardless of success +*/ +static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ) +{ + int32_t oldval, res; + + __TBB_full_memory_fence(); + + do { + __asm__ __volatile__( + "ldrex %1, [%3]\n" + "mov %0, #0\n" + "cmp %1, %4\n" + "it eq\n" + "strexeq %0, %5, [%3]\n" + : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int32_t*)ptr) + : "r" ((volatile int32_t *)ptr), "Ir" (comparand), "r" (value) + : "cc"); + } while (res); + + __TBB_full_memory_fence(); + + return oldval; +} + +/** + * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr + * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand + * @param value value to assign *ptr to if *ptr==comparand + * @param comparand value to compare with *ptr + * @return value originally in memory at ptr, regardless of success + */ +static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ) +{ + int64_t oldval; + int32_t res; + + __TBB_full_memory_fence(); + + do { + __asm__ __volatile__( + "mov %0, #0\n" + "ldrexd %1, %H1, [%3]\n" + "cmp %1, %4\n" + "it eq\n" + "cmpeq %H1, %H4\n" + "it eq\n" + "strexdeq %0, %5, %H5, [%3]" + : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int64_t*)ptr) + : "r" ((volatile int64_t *)ptr), "r" (comparand), "r" (value) + : "cc"); + } while (res); + + __TBB_full_memory_fence(); + + return oldval; +} + +static inline int32_t __TBB_machine_fetchadd4(volatile void* ptr, int32_t addend) +{ + unsigned long tmp; + int32_t result, tmp2; + + __TBB_full_memory_fence(); + + __asm__ __volatile__( +"1: ldrex %0, [%4]\n" +" add %3, %0, %5\n" +" strex %1, %3, [%4]\n" +" cmp %1, #0\n" +" bne 1b\n" + : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int32_t*)ptr), "=&r"(tmp2) + : "r" ((volatile int32_t *)ptr), "Ir" (addend) + : "cc"); + + __TBB_full_memory_fence(); + + return result; +} + +static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) +{ + unsigned long tmp; + int64_t result, tmp2; + + __TBB_full_memory_fence(); + + __asm__ __volatile__( +"1: ldrexd %0, %H0, [%4]\n" +" adds %3, %0, %5\n" +" adc %H3, %H0, %H5\n" +" strexd %1, %3, %H3, [%4]\n" +" cmp %1, #0\n" +" bne 1b" + : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int64_t*)ptr), "=&r"(tmp2) + : "r" ((volatile int64_t *)ptr), "r" (addend) + : "cc"); + + + __TBB_full_memory_fence(); + + return result; +} + +namespace tbb { +namespace internal { + template + struct machine_load_store_relaxed { + static inline T load ( const volatile T& location ) { + const T value = location; + + /* + * An extra memory barrier is required for errata #761319 + * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a + */ + __TBB_acquire_consistency_helper(); + return value; + } + + static inline void store ( volatile T& location, T value ) { + location = value; + } + }; +}} // namespaces internal, tbb + +// Machine specific atomic operations + +#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) +#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) + +// Use generics for some things +#define __TBB_USE_GENERIC_PART_WORD_CAS 1 +#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 +#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 +#define __TBB_USE_GENERIC_FETCH_STORE 1 +#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 +#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 +#elif defined __aarch64__ +// Generic gcc implementations are fine for ARMv8-a except __TBB_PAUSE. +#include "gcc_generic.h" +#else +#error compilation requires an ARMv7-a or ARMv8-a architecture. +#endif // __ARM_ARCH_7A__ + +inline void __TBB_machine_pause (int32_t delay) +{ + while(delay>0) + { + __asm__ __volatile__("yield" ::: "memory"); + delay--; + } +} +#define __TBB_Pause(V) __TBB_machine_pause(V) diff --git a/inst/include/tbb_local/tbb/machine/gcc_generic.h b/inst/include/tbb_local/tbb/machine/gcc_generic.h index 5fc2a901..cbf8d993 100644 --- a/inst/include/tbb_local/tbb/machine/gcc_generic.h +++ b/inst/include/tbb_local/tbb/machine/gcc_generic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_generic_H) @@ -106,23 +102,17 @@ __TBB_MACHINE_DEFINE_ATOMICS(8,int64_t) #undef __TBB_MACHINE_DEFINE_ATOMICS -namespace tbb{ namespace internal { namespace gcc_builtins { - inline int clz(unsigned int x){ return __builtin_clz(x);}; - inline int clz(unsigned long int x){ return __builtin_clzl(x);}; - inline int clz(unsigned long long int x){ return __builtin_clzll(x);}; -}}} -//gcc __builtin_clz builtin count _number_ of leading zeroes -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - return sizeof(x)*8 - tbb::internal::gcc_builtins::clz(x) -1 ; -} - - typedef unsigned char __TBB_Flag; typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; #if __TBB_GCC_VERSION < 40700 // Use __sync_* builtins +// Use generic machine_load_store functions if there are no builtin atomics +#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 + static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { __sync_fetch_and_or(reinterpret_cast(ptr),addend); } @@ -158,6 +148,59 @@ inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { __atomic_clear(&flag,__ATOMIC_RELEASE); } +namespace tbb { namespace internal { + +/** GCC atomic operation intrinsics might miss compiler fence. + Adding it after load-with-acquire, before store-with-release, and + on both sides of sequentially consistent operations is sufficient for correctness. **/ + +template +inline T __TBB_machine_atomic_load( const volatile T& location) { + if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); + T value = __atomic_load_n(&location, MemOrder); + if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); + return value; +} + +template +inline void __TBB_machine_atomic_store( volatile T& location, T value) { + if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); + __atomic_store_n(&location, value, MemOrder); + if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); +} + +template +struct machine_load_store { + static T load_with_acquire ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static void store_with_release ( volatile T &location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +template +struct machine_load_store_relaxed { + static inline T load ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static inline void store ( volatile T& location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +template +struct machine_load_store_seq_cst { + static T load ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static void store ( volatile T &location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +}} // namespace tbb::internal + #endif // __TBB_GCC_VERSION < 40700 // Machine specific atomic operations @@ -167,18 +210,24 @@ inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { #define __TBB_TryLockByte __TBB_machine_try_lock_byte #define __TBB_UnlockByte __TBB_machine_unlock_byte -// Definition of other functions -#define __TBB_Log2(V) __TBB_machine_lg(V) +// __builtin_clz counts the number of leading zeroes +namespace tbb{ namespace internal { namespace gcc_builtins { + inline int clz(unsigned int x){ return __builtin_clz(x); } + inline int clz(unsigned long int x){ return __builtin_clzl(x); } + inline int clz(unsigned long long int x){ return __builtin_clzll(x); } +}}} +// logarithm is the index of the most significant non-zero bit +static inline intptr_t __TBB_machine_lg( uintptr_t x ) { + // If P is a power of 2 and x static inline intptr_t __TBB_machine_lg( T x ) { __TBB_ASSERT(x>0, "The logarithm of a non-positive value is undefined."); @@ -33,6 +29,7 @@ static inline intptr_t __TBB_machine_lg( T x ) { return j; } #define __TBB_Log2(V) __TBB_machine_lg(V) +#endif /* !__TBB_Log2 */ #ifndef __TBB_Pause //TODO: check if raising a ratio of pause instructions to loop control instructions diff --git a/inst/include/tbb_local/tbb/machine/gcc_itsx.h b/inst/include/tbb_local/tbb/machine/gcc_itsx.h index caa35441..5bd400e5 100644 --- a/inst/include/tbb_local/tbb/machine/gcc_itsx.h +++ b/inst/include/tbb_local/tbb/machine/gcc_itsx.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_itsx_H) diff --git a/inst/include/tbb_local/tbb/machine/ibm_aix51.h b/inst/include/tbb_local/tbb/machine/ibm_aix51.h index a905b4e1..14ba4d9f 100644 --- a/inst/include/tbb_local/tbb/machine/ibm_aix51.h +++ b/inst/include/tbb_local/tbb/machine/ibm_aix51.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // TODO: revise by comparing with mac_ppc.h diff --git a/inst/include/tbb_local/tbb/machine/icc_generic.h b/inst/include/tbb_local/tbb/machine/icc_generic.h index 04863000..00af78a7 100644 --- a/inst/include/tbb_local/tbb/machine/icc_generic.h +++ b/inst/include/tbb_local/tbb/machine/icc_generic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_icc_generic_H) @@ -23,7 +19,7 @@ #endif #if ! __TBB_ICC_BUILTIN_ATOMICS_PRESENT - #error "Intel C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" + #error "Intel(R) C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" #endif #define __TBB_machine_icc_generic_H diff --git a/inst/include/tbb_local/tbb/machine/linux_common.h b/inst/include/tbb_local/tbb/machine/linux_common.h index 4d2d355b..6f93828d 100644 --- a/inst/include/tbb_local/tbb/machine/linux_common.h +++ b/inst/include/tbb_local/tbb/machine/linux_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_H @@ -30,19 +26,44 @@ #include #if defined(SYS_futex) +/* This header file is included for Linux and some other systems that may support futexes.*/ #define __TBB_USE_FUTEX 1 + +#if defined(__has_include) +#define __TBB_has_include __has_include +#else +#define __TBB_has_include(x) 0 +#endif + +/* +If available, use typical headers where futex API is defined. While Linux and OpenBSD +are known to provide such headers, other systems might have them as well. +*/ +#if defined(__linux__) || __TBB_has_include() +#include +#elif defined(__OpenBSD__) || __TBB_has_include() +#include +#endif + #include #include -// Unfortunately, some versions of Linux do not have a header that defines FUTEX_WAIT and FUTEX_WAKE. -#ifdef FUTEX_WAIT +/* +Some systems might not define the macros or use different names. In such case we expect +the actual parameter values to match Linux: 0 for wait, 1 for wake. +*/ +#if defined(FUTEX_WAIT_PRIVATE) +#define __TBB_FUTEX_WAIT FUTEX_WAIT_PRIVATE +#elif defined(FUTEX_WAIT) #define __TBB_FUTEX_WAIT FUTEX_WAIT #else #define __TBB_FUTEX_WAIT 0 #endif -#ifdef FUTEX_WAKE +#if defined(FUTEX_WAKE_PRIVATE) +#define __TBB_FUTEX_WAKE FUTEX_WAKE_PRIVATE +#elif defined(FUTEX_WAKE) #define __TBB_FUTEX_WAKE FUTEX_WAKE #else #define __TBB_FUTEX_WAKE 1 diff --git a/inst/include/tbb_local/tbb/machine/linux_ia32.h b/inst/include/tbb_local/tbb/machine/linux_ia32.h index 932d343d..6e2543c7 100644 --- a/inst/include/tbb_local/tbb/machine/linux_ia32.h +++ b/inst/include/tbb_local/tbb/machine/linux_ia32.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia32_H) diff --git a/inst/include/tbb_local/tbb/machine/linux_ia64.h b/inst/include/tbb_local/tbb/machine/linux_ia64.h index f4772288..be9676e1 100644 --- a/inst/include/tbb_local/tbb/machine/linux_ia64.h +++ b/inst/include/tbb_local/tbb/machine/linux_ia64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia64_H) diff --git a/inst/include/tbb_local/tbb/machine/linux_intel64.h b/inst/include/tbb_local/tbb/machine/linux_intel64.h index 02153c2a..d0b15bed 100644 --- a/inst/include/tbb_local/tbb/machine/linux_intel64.h +++ b/inst/include/tbb_local/tbb/machine/linux_intel64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_intel64_H) diff --git a/inst/include/tbb_local/tbb/machine/mac_ppc.h b/inst/include/tbb_local/tbb/machine/mac_ppc.h index 13f387b4..0c07cc99 100644 --- a/inst/include/tbb_local/tbb/machine/mac_ppc.h +++ b/inst/include/tbb_local/tbb/machine/mac_ppc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_power_H) diff --git a/inst/include/tbb_local/tbb/machine/macos_common.h b/inst/include/tbb_local/tbb/machine/macos_common.h index 119ad979..c63536fa 100644 --- a/inst/include/tbb_local/tbb/machine/macos_common.h +++ b/inst/include/tbb_local/tbb/machine/macos_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_macos_common_H) diff --git a/inst/include/tbb_local/tbb/machine/mic_common.h b/inst/include/tbb_local/tbb/machine/mic_common.h index 8765d39f..afe08331 100644 --- a/inst/include/tbb_local/tbb/machine/mic_common.h +++ b/inst/include/tbb_local/tbb/machine/mic_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mic_common_H diff --git a/inst/include/tbb_local/tbb/machine/msvc_armv7.h b/inst/include/tbb_local/tbb/machine/msvc_armv7.h index 40d22020..d38ceb9c 100644 --- a/inst/include/tbb_local/tbb/machine/msvc_armv7.h +++ b/inst/include/tbb_local/tbb/machine/msvc_armv7.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_msvc_armv7_H) diff --git a/inst/include/tbb_local/tbb/machine/msvc_ia32_common.h b/inst/include/tbb_local/tbb/machine/msvc_ia32_common.h index 8b4814bc..541e1fa9 100644 --- a/inst/include/tbb_local/tbb/machine/msvc_ia32_common.h +++ b/inst/include/tbb_local/tbb/machine/msvc_ia32_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_msvc_ia32_common_H) @@ -49,7 +45,7 @@ }}} #endif -#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) +#if __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT // S is the operand size in bytes, B is the suffix for intrinsics for that size #define __TBB_MACHINE_DEFINE_ATOMICS(S,B,T,U) \ __pragma(intrinsic( _InterlockedCompareExchange##B )) \ @@ -75,8 +71,7 @@ #endif #undef __TBB_MACHINE_DEFINE_ATOMICS - #define __TBB_ATOMIC_PRIMITIVES_DEFINED -#endif /*_MSC_VER>=1600*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ #if _MSC_VER>=1300 || __INTEL_COMPILER>=1100 #pragma intrinsic(_ReadWriteBarrier) diff --git a/inst/include/tbb_local/tbb/machine/sunos_sparc.h b/inst/include/tbb_local/tbb/machine/sunos_sparc.h index 9119f402..18e68c34 100644 --- a/inst/include/tbb_local/tbb/machine/sunos_sparc.h +++ b/inst/include/tbb_local/tbb/machine/sunos_sparc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ diff --git a/inst/include/tbb_local/tbb/machine/windows_api.h b/inst/include/tbb_local/tbb/machine/windows_api.h index d362abc5..abf0e182 100644 --- a/inst/include/tbb_local/tbb/machine/windows_api.h +++ b/inst/include/tbb_local/tbb/machine/windows_api.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_windows_api_H diff --git a/inst/include/tbb_local/tbb/machine/windows_ia32.h b/inst/include/tbb_local/tbb/machine/windows_ia32.h index 8db0d2b8..86550a7e 100644 --- a/inst/include/tbb_local/tbb/machine/windows_ia32.h +++ b/inst/include/tbb_local/tbb/machine/windows_ia32.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_ia32_H) @@ -43,7 +39,7 @@ extern "C" { __int64 __TBB_EXPORTED_FUNC __TBB_machine_load8 (const volatile void *ptr); } -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED +#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT #define __TBB_MACHINE_DEFINE_ATOMICS(S,T,U,A,C) \ static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ @@ -93,7 +89,7 @@ __TBB_MACHINE_DEFINE_ATOMICS(4, ptrdiff_t, ptrdiff_t, eax, ecx) #undef __TBB_MACHINE_DEFINE_ATOMICS -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ //TODO: Check if it possible and profitable for IA-32 architecture on (Linux and Windows) //to use of 64-bit load/store via floating point registers together with full fence diff --git a/inst/include/tbb_local/tbb/machine/windows_intel64.h b/inst/include/tbb_local/tbb/machine/windows_intel64.h index 86abd6ad..6b733d20 100644 --- a/inst/include/tbb_local/tbb/machine/windows_intel64.h +++ b/inst/include/tbb_local/tbb/machine/windows_intel64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_intel64_H) @@ -29,7 +25,7 @@ #include "msvc_ia32_common.h" -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED +#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT #include #pragma intrinsic(_InterlockedCompareExchange,_InterlockedExchangeAdd,_InterlockedExchange) @@ -66,7 +62,7 @@ inline __int64 __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ) { return _InterlockedExchange64( (__int64*)ptr, value ); } -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ #define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 #define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 diff --git a/inst/include/tbb_local/tbb/machine/xbox360_ppc.h b/inst/include/tbb_local/tbb/machine/xbox360_ppc.h deleted file mode 100644 index 148e5b1d..00000000 --- a/inst/include/tbb_local/tbb/machine/xbox360_ppc.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - Copyright 2005-2014 Intel Corporation. All Rights Reserved. - - This file is part of Threading Building Blocks. Threading Building Blocks is free software; - you can redistribute it and/or modify it under the terms of the GNU General Public License - version 2 as published by the Free Software Foundation. Threading Building Blocks is - distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the - implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. You should have received a copy of - the GNU General Public License along with Threading Building Blocks; if not, write to the - Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - As a special exception, you may use this file as part of a free software library without - restriction. Specifically, if other files instantiate templates or use macros or inline - functions from this file, or you compile this file and link it with other files to produce - an executable, this file does not by itself cause the resulting executable to be covered - by the GNU General Public License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General Public License. -*/ - -// TODO: revise by comparing with mac_ppc.h - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_xbox360_ppc_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_xbox360_ppc_H - -#define NONET -#define NOD3D -#include "xtl.h" -#include "ppcintrinsics.h" - -#if _MSC_VER >= 1300 -extern "C" void _MemoryBarrier(); -#pragma intrinsic(_MemoryBarrier) -#define __TBB_control_consistency_helper() __isync() -#define __TBB_acquire_consistency_helper() _MemoryBarrier() -#define __TBB_release_consistency_helper() _MemoryBarrier() -#endif - -#define __TBB_full_memory_fence() __sync() - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG - -//todo: define __TBB_USE_FENCED_ATOMICS and define acquire/release primitives to maximize performance - -inline __int32 __TBB_machine_cmpswp4(volatile void *ptr, __int32 value, __int32 comparand ) { - __sync(); - __int32 result = InterlockedCompareExchange((volatile LONG*)ptr, value, comparand); - __isync(); - return result; -} - -inline __int64 __TBB_machine_cmpswp8(volatile void *ptr, __int64 value, __int64 comparand ) -{ - __sync(); - __int64 result = InterlockedCompareExchange64((volatile LONG64*)ptr, value, comparand); - __isync(); - return result; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#pragma optimize( "", off ) -inline void __TBB_machine_pause (__int32 delay ) -{ - for (__int32 i=0; i> 0) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 1) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 2) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 3) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 4) & 1) + - ((__TBB_XBOX360_HARDWARE_THREAD_MASK >> 5) & 1) + 1; // +1 accomodates for the master thread -} - -static inline int __TBB_XBOX360_GetHardwareThreadIndex(int workerThreadIndex) -{ - workerThreadIndex %= __TBB_XBOX360_DetectNumberOfWorkers()-1; - int m = __TBB_XBOX360_HARDWARE_THREAD_MASK; - int index = 0; - int skipcount = workerThreadIndex; - while (true) - { - if ((m & 1)!=0) - { - if (skipcount==0) break; - skipcount--; - } - m >>= 1; - index++; - } - return index; -} - -#define __TBB_HardwareConcurrency() __TBB_XBOX360_DetectNumberOfWorkers() diff --git a/inst/include/tbb_local/tbb/memory_pool.h b/inst/include/tbb_local/tbb/memory_pool.h index b3bba397..2037d634 100644 --- a/inst/include/tbb_local/tbb/memory_pool.h +++ b/inst/include/tbb_local/tbb/memory_pool.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_memory_pool_H diff --git a/inst/include/tbb_local/tbb/mutex.h b/inst/include/tbb_local/tbb/mutex.h index e40b4cd0..5f461f3f 100644 --- a/inst/include/tbb_local/tbb/mutex.h +++ b/inst/include/tbb_local/tbb/mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mutex_H diff --git a/inst/include/tbb_local/tbb/null_mutex.h b/inst/include/tbb_local/tbb/null_mutex.h index 85c660e3..b9c382dd 100644 --- a/inst/include/tbb_local/tbb/null_mutex.h +++ b/inst/include/tbb_local/tbb/null_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_null_mutex_H diff --git a/inst/include/tbb_local/tbb/null_rw_mutex.h b/inst/include/tbb_local/tbb/null_rw_mutex.h index fa0c8035..d2fd33ca 100644 --- a/inst/include/tbb_local/tbb/null_rw_mutex.h +++ b/inst/include/tbb_local/tbb/null_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_null_rw_mutex_H diff --git a/inst/include/tbb_local/tbb/parallel_do.h b/inst/include/tbb_local/tbb/parallel_do.h index 15275682..d3f2075d 100644 --- a/inst/include/tbb_local/tbb/parallel_do.h +++ b/inst/include/tbb_local/tbb/parallel_do.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_do_H @@ -500,7 +496,7 @@ void parallel_do( Iterator first, Iterator last, const Body& body ) if ( first == last ) return; #if __TBB_TASK_GROUP_CONTEXT - task_group_context context; + task_group_context context(internal::PARALLEL_DO); #endif interface9::internal::select_parallel_do( first, last, body, &Body::operator() #if __TBB_TASK_GROUP_CONTEXT diff --git a/inst/include/tbb_local/tbb/parallel_for.h b/inst/include/tbb_local/tbb/parallel_for.h index 2cab6586..a682067a 100644 --- a/inst/include/tbb_local/tbb/parallel_for.h +++ b/inst/include/tbb_local/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_for_H @@ -26,6 +22,7 @@ #include "partitioner.h" #include "blocked_range.h" #include "tbb_exception.h" +#include "internal/_tbb_trace_impl.h" namespace tbb { @@ -57,6 +54,7 @@ namespace internal { my_body(body), my_partition(partitioner) { + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, NULL); } //! Splitting constructor used to generate children. /** parent_ becomes left child. Newly constructed object is right child. */ @@ -66,6 +64,7 @@ namespace internal { my_partition(parent_.my_partition, split_obj) { my_partition.set_affinity(*this); + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); } //! Construct right child from the given range as response to the demand. /** parent_ remains left child. Newly constructed object is right child. */ @@ -76,6 +75,7 @@ namespace internal { { my_partition.set_affinity(*this); my_partition.align_depth( d ); + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); } static void run( const Range& range, const Body& body, Partitioner& partitioner ) { if( !range.empty() ) { @@ -84,22 +84,34 @@ namespace internal { #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; + task_group_context context(PARALLEL_FOR); start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ + // REGION BEGIN + fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); task::spawn_root_and_wait(a); + fgt_end_algorithm( (void*)&context ); + // REGION END } } #if __TBB_TASK_GROUP_CONTEXT static void run( const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context ) { if( !range.empty() ) { start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); + // REGION BEGIN + fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); task::spawn_root_and_wait(a); + fgt_end_algorithm( (void*)&context ); + // END REGION } } #endif /* __TBB_TASK_GROUP_CONTEXT */ //! Run body for range, serves as callback for partitioner - void run_body( Range &r ) { my_body( r ); } + void run_body( Range &r ) { + fgt_alg_begin_body( tbb::internal::PARALLEL_FOR_TASK, (void *)const_cast(&(this->my_body)), (void*)this ); + my_body( r ); + fgt_alg_end_body( (void *)const_cast(&(this->my_body)) ); + } //! spawn right task, serves as callback for partitioner void offer_work(typename Partitioner::split_type& split_obj) { diff --git a/inst/include/tbb_local/tbb/parallel_for_each.h b/inst/include/tbb_local/tbb/parallel_for_each.h index 6c2ec9f0..1ce39ece 100644 --- a/inst/include/tbb_local/tbb/parallel_for_each.h +++ b/inst/include/tbb_local/tbb/parallel_for_each.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_for_each_H diff --git a/inst/include/tbb_local/tbb/parallel_invoke.h b/inst/include/tbb_local/tbb/parallel_invoke.h index 0dd7590e..026b445d 100644 --- a/inst/include/tbb_local/tbb/parallel_invoke.h +++ b/inst/include/tbb_local/tbb/parallel_invoke.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_invoke_H #define __TBB_parallel_invoke_H #include "task.h" +#include "tbb_profiling.h" #if __TBB_VARIADIC_PARALLEL_INVOKE #include // std::forward @@ -31,7 +28,9 @@ namespace tbb { #if !__TBB_TASK_GROUP_CONTEXT /** Dummy to avoid cluttering the bulk of the header with enormous amount of ifdefs. **/ - struct task_group_context {}; + struct task_group_context { + task_group_context(tbb::internal::string_index){} + }; #endif /* __TBB_TASK_GROUP_CONTEXT */ //! @cond INTERNAL @@ -223,7 +222,7 @@ namespace internal { // task_group_context is not in the pack, needs to be added template void parallel_invoke_impl(false_type, F0&& f0, F1&& f1, F&&... f) { - tbb::task_group_context context; + tbb::task_group_context context(PARALLEL_INVOKE); // Add context to the arguments, and redirect to the other overload parallel_invoke_impl(true_type(), std::forward(f0), std::forward(f1), std::forward(f)..., context); } @@ -385,31 +384,31 @@ void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, con // two arguments template void parallel_invoke(const F0& f0, const F1& f1) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, context); } // three arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, context); } // four arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, context); } // five arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, f4, context); } // six arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, f4, f5, context); } // seven arguments @@ -417,7 +416,7 @@ template(f0, f1, f2, f3, f4, f5, f6, context); } // eight arguments @@ -426,7 +425,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, context); } // nine arguments @@ -435,7 +434,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, f8, context); } // ten arguments @@ -444,7 +443,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, context); } #endif // __TBB_VARIADIC_PARALLEL_INVOKE diff --git a/inst/include/tbb_local/tbb/parallel_reduce.h b/inst/include/tbb_local/tbb/parallel_reduce.h index 0596ae03..9908de9e 100644 --- a/inst/include/tbb_local/tbb/parallel_reduce.h +++ b/inst/include/tbb_local/tbb/parallel_reduce.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_reduce_H @@ -136,7 +132,7 @@ namespace internal { #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; + task_group_context context(PARALLEL_REDUCE); task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ } @@ -213,67 +209,71 @@ namespace internal { my_left_body.join( my_right_body ); return NULL; } - template + template friend class start_deterministic_reduce; }; //! Task type used to split the work of parallel_deterministic_reduce. /** @ingroup algorithms */ - template + template class start_deterministic_reduce: public task { typedef finish_deterministic_reduce finish_type; Body &my_body; Range my_range; + typename Partitioner::task_partition_type my_partition; task* execute() __TBB_override; //! Constructor used for root task - start_deterministic_reduce( const Range& range, Body& body ) : + start_deterministic_reduce( const Range& range, Body& body, Partitioner& partitioner ) : my_body( body ), - my_range( range ) + my_range( range ), + my_partition( partitioner ) { } //! Splitting constructor used to generate children. /** parent_ becomes left child. Newly constructed object is right child. */ - start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c ) : + start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c, typename Partitioner::split_type& split_obj ) : my_body( c.my_right_body ), - my_range( parent_.my_range, split() ) + my_range( parent_.my_range, split_obj ), + my_partition( parent_.my_partition, split_obj ) { } public: - static void run( const Range& range, Body& body ) { + static void run( const Range& range, Body& body, Partitioner& partitioner ) { if( !range.empty() ) { #if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_deterministic_reduce(range,&body) ); + task::spawn_root_and_wait( *new(task::allocate_root()) start_deterministic_reduce(range,&body,partitioner) ); #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body) ); + task_group_context context(PARALLEL_REDUCE); + task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ } } #if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, task_group_context& context ) { + static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body) ); + task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); } #endif /* __TBB_TASK_GROUP_CONTEXT */ - }; - template - task* start_deterministic_reduce::execute() { - if( !my_range.is_divisible() ) { - my_body( my_range ); - return NULL; - } else { - finish_type& c = *new( allocate_continuation() ) finish_type( my_body ); - recycle_as_child_of(c); - c.set_ref_count(2); - start_deterministic_reduce& b = *new( c.allocate_child() ) start_deterministic_reduce( *this, c ); - task::spawn(b); - return this; + void offer_work( typename Partitioner::split_type& split_obj) { + task* tasks[2]; + allocate_sibling(static_cast(this), tasks, sizeof(start_deterministic_reduce), sizeof(finish_type)); + new((void*)tasks[0]) finish_type(my_body); + new((void*)tasks[1]) start_deterministic_reduce(*this, *static_cast(tasks[0]), split_obj); + spawn(*tasks[1]); } + + void run_body( Range &r ) { my_body(r); } + }; + + template + task* start_deterministic_reduce::execute() { + my_partition.execute(*this, my_range); + return NULL; } } // namespace internal //! @endcond @@ -389,6 +389,13 @@ void parallel_reduce( const Range& range, Body& body, affinity_partitioner& part } #if __TBB_TASK_GROUP_CONTEXT +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +void parallel_reduce( const Range& range, Body& body, task_group_context& context ) { + internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); +} + //! Parallel iteration with reduction, simple partitioner and user-supplied context. /** @ingroup algorithms **/ template @@ -476,6 +483,17 @@ Value parallel_reduce( const Range& range, const Value& identity, const RealBody } #if __TBB_TASK_GROUP_CONTEXT +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + internal::lambda_reduce_body body(identity, real_body, reduction); + internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); + return body.result(); +} + //! Parallel iteration with reduction, simple partitioner and user-supplied context. /** @ingroup algorithms **/ template @@ -521,44 +539,108 @@ Value parallel_reduce( const Range& range, const Value& identity, const RealBody } #endif /* __TBB_TASK_GROUP_CONTEXT */ -//! Parallel iteration with deterministic reduction and default partitioner. +//! Parallel iteration with deterministic reduction and default simple partitioner. /** @ingroup algorithms **/ template void parallel_deterministic_reduce( const Range& range, Body& body ) { - internal::start_deterministic_reduce::run( range, body ); + internal::start_deterministic_reduce::run(range, body, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { + internal::start_deterministic_reduce::run(range, body, partitioner); +} + +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { + internal::start_deterministic_reduce::run(range, body, partitioner); } #if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. /** @ingroup algorithms **/ template void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { - internal::start_deterministic_reduce::run( range, body, context ); + internal::start_deterministic_reduce::run( range, body, simple_partitioner(), context ); +} + +//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + internal::start_deterministic_reduce::run(range, body, partitioner, context); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { + internal::start_deterministic_reduce::run(range, body, partitioner, context); } #endif /* __TBB_TASK_GROUP_CONTEXT */ /** parallel_reduce overloads that work with anonymous function objects (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ -//! Parallel iteration with deterministic reduction and default partitioner. +//! Parallel iteration with deterministic reduction and default simple partitioner. +// TODO: consider making static_partitioner the default /** @ingroup algorithms **/ template Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) { internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce > - ::run(range, body); + internal::start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner); return body.result(); } +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) { + internal::lambda_reduce_body body(identity, real_body, reduction); + internal::start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner); + return body.result(); +} #if __TBB_TASK_GROUP_CONTEXT +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context); +} + //! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. /** @ingroup algorithms **/ template Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce > - ::run( range, body, context ); + const simple_partitioner& partitioner, task_group_context& context ) { + internal::lambda_reduce_body body(identity, real_body, reduction); + internal::start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner, context); + return body.result(); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner, task_group_context& context ) { + internal::lambda_reduce_body body(identity, real_body, reduction); + internal::start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner, context); return body.result(); } #endif /* __TBB_TASK_GROUP_CONTEXT */ diff --git a/inst/include/tbb_local/tbb/parallel_scan.h b/inst/include/tbb_local/tbb/parallel_scan.h index faf6b316..7f6478da 100644 --- a/inst/include/tbb_local/tbb/parallel_scan.h +++ b/inst/include/tbb_local/tbb/parallel_scan.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_scan_H @@ -32,12 +28,14 @@ namespace tbb { /** @ingroup algorithms */ struct pre_scan_tag { static bool is_final_scan() {return false;} + operator bool() {return is_final_scan();} }; //! Used to indicate that the final scan is being performed. /** @ingroup algorithms */ struct final_scan_tag { static bool is_final_scan() {return true;} + operator bool() {return is_final_scan();} }; //! @cond INTERNAL @@ -227,7 +225,6 @@ namespace internal { if( !range_.empty() ) { typedef internal::start_scan start_pass1_type; internal::sum_node* root = NULL; - typedef internal::final_sum final_sum_type; final_sum_type* temp_body = new(task::allocate_root()) final_sum_type( body_ ); start_pass1_type& pass1 = *new(task::allocate_root()) start_pass1_type( /*my_return_slot=*/root, @@ -297,6 +294,43 @@ namespace internal { } return next_task; } + + template + class lambda_scan_body : no_assign { + Value my_sum; + const Value& identity_element; + const Scan& my_scan; + const ReverseJoin& my_reverse_join; + public: + lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join) + : my_sum(identity) + , identity_element(identity) + , my_scan(scan) + , my_reverse_join(rev_join) {} + + lambda_scan_body( lambda_scan_body& b, split ) + : my_sum(b.identity_element) + , identity_element(b.identity_element) + , my_scan(b.my_scan) + , my_reverse_join(b.my_reverse_join) {} + + template + void operator()( const Range& r, Tag tag ) { + my_sum = my_scan(r, my_sum, tag); + } + + void reverse_join( lambda_scan_body& a ) { + my_sum = my_reverse_join(a.my_sum, my_sum); + } + + void assign( lambda_scan_body& b ) { + my_sum = b.my_sum; + } + + Value result() const { + return my_sum; + } + }; } // namespace internal //! @endcond @@ -340,6 +374,34 @@ template void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { internal::start_scan::run(range,body,partitioner); } + +//! Parallel prefix with default partitioner +/** @ingroup algorithms **/ +template +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) { + internal::lambda_scan_body body(identity, scan, reverse_join); + tbb::parallel_scan(range,body,__TBB_DEFAULT_PARTITIONER()); + return body.result(); +} + +//! Parallel prefix with simple_partitioner +/** @ingroup algorithms **/ +template +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const simple_partitioner& partitioner ) { + internal::lambda_scan_body body(identity, scan, reverse_join); + tbb::parallel_scan(range,body,partitioner); + return body.result(); +} + +//! Parallel prefix with auto_partitioner +/** @ingroup algorithms **/ +template +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const auto_partitioner& partitioner ) { + internal::lambda_scan_body body(identity, scan, reverse_join); + tbb::parallel_scan(range,body,partitioner); + return body.result(); +} + //@} } // namespace tbb diff --git a/inst/include/tbb_local/tbb/parallel_sort.h b/inst/include/tbb_local/tbb/parallel_sort.h index 07d3907b..a543a6aa 100644 --- a/inst/include/tbb_local/tbb/parallel_sort.h +++ b/inst/include/tbb_local/tbb/parallel_sort.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_sort_H @@ -27,6 +23,9 @@ #include #include #include +#if __TBB_TASK_GROUP_CONTEXT + #include "tbb_profiling.h" +#endif namespace tbb { @@ -159,7 +158,7 @@ struct quick_sort_body { template void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { #if __TBB_TASK_GROUP_CONTEXT - task_group_context my_context; + task_group_context my_context(PARALLEL_SORT); const int serial_cutoff = 9; __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); @@ -230,13 +229,6 @@ void parallel_sort(Range& rng, const Compare& comp) { parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); } -//! Sorts the data in const rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - //! Sorts the data in rng with a default comparator \c std::less /** @ingroup algorithms **/ template @@ -244,13 +236,6 @@ void parallel_sort(Range& rng) { parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); } -//! Sorts the data in const rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - //! Sorts the data in the range \c [begin,end) with a default comparator \c std::less /** @ingroup algorithms **/ template diff --git a/inst/include/tbb_local/tbb/parallel_while.h b/inst/include/tbb_local/tbb/parallel_while.h index 2f37a41a..07177005 100644 --- a/inst/include/tbb_local/tbb/parallel_while.h +++ b/inst/include/tbb_local/tbb/parallel_while.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_while diff --git a/inst/include/tbb_local/tbb/partitioner.h b/inst/include/tbb_local/tbb/partitioner.h index 96a0b757..285ada71 100644 --- a/inst/include/tbb_local/tbb/partitioner.h +++ b/inst/include/tbb_local/tbb/partitioner.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_partitioner_H @@ -50,6 +46,7 @@ #endif // __TBB_DEFINE_MIC #include "task.h" +#include "task_arena.h" #include "aligned_space.h" #include "atomic.h" #include "internal/_template_helpers.h" @@ -122,6 +119,7 @@ namespace internal { using namespace tbb::internal; template class start_for; template class start_reduce; +template class start_deterministic_reduce; //! Join task node that contains shared flag for stealing feedback class flag_task: public task { @@ -260,7 +258,6 @@ struct partition_type_base { template struct adaptive_mode : partition_type_base { typedef Partition my_partition; - using partition_type_base::self; // CRTP helper to get access to derived classes size_t my_divisor; // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves. // A task which has only one index must produce the right split without reserved index in order to avoid @@ -269,12 +266,44 @@ struct adaptive_mode : partition_type_base { static const unsigned factor = 1; adaptive_mode() : my_divisor(tbb::internal::get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {} - adaptive_mode(adaptive_mode &src, const proportional_split& split_obj) : my_divisor(do_split(src, split_obj)) {} /*! Override do_split methods in order to specify splitting strategy */ size_t do_split(adaptive_mode &src, split) { return src.my_divisor /= 2u; } - size_t do_split(adaptive_mode &src, const proportional_split& split_obj) { +}; + +//! A helper class to create a proportional_split object for a given type of Range. +/** If the Range has static boolean constant 'is_splittable_in_proportion' set to 'true', + the created object splits a provided value in an implemenation-defined proportion; + otherwise it represents equal-size split. */ +// TODO: check if this helper can be a nested class of proportional_mode. +template +struct proportion_helper { + static proportional_split get_split(size_t) { return proportional_split(1,1); } +}; +template +struct proportion_helper::type> { + static proportional_split get_split(size_t n) { +#if __TBB_NONUNIFORM_TASK_CREATION + size_t right = (n + 2) / 3; +#else + size_t right = n / 2; +#endif + size_t left = n - right; + return proportional_split(left, right); + } +}; + +//! Provides proportional splitting strategy for partition objects +template +struct proportional_mode : adaptive_mode { + typedef Partition my_partition; + using partition_type_base::self; // CRTP helper to get access to derived classes + + proportional_mode() : adaptive_mode() {} + proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {} + proportional_mode(proportional_mode &src, const proportional_split& split_obj) { self().my_divisor = do_split(src, split_obj); } + size_t do_split(proportional_mode &src, const proportional_split& split_obj) { #if __TBB_ENABLE_RANGE_FEEDBACK size_t portion = size_t(float(src.my_divisor) * float(split_obj.right()) / float(split_obj.left() + split_obj.right()) + 0.5f); @@ -293,81 +322,43 @@ struct adaptive_mode : partition_type_base { return portion; } bool is_divisible() { // part of old should_execute_range() - return my_divisor > my_partition::factor; + return self().my_divisor > my_partition::factor; + } + template + proportional_split get_split() { + // Create a proportion for the number of threads expected to handle "this" subrange + return proportion_helper::get_split( self().my_divisor / my_partition::factor ); } }; +static size_t get_initial_partition_head() { + int current_index = tbb::this_task_arena::current_thread_index(); + if (current_index == tbb::task_arena::not_initialized) + current_index = 0; + return size_t(current_index); +} + //! Provides default linear indexing of partitioner's sequence template -struct linear_affinity_mode : adaptive_mode { - using adaptive_mode::my_divisor; +struct linear_affinity_mode : proportional_mode { size_t my_head; - using adaptive_mode::self; - linear_affinity_mode() : adaptive_mode(), my_head(0) {} - linear_affinity_mode(linear_affinity_mode &src, split) : adaptive_mode(src, split()) - , my_head(src.my_head + src.my_divisor) {} - linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : adaptive_mode(src, split_obj) - , my_head(src.my_head + src.my_divisor) {} + size_t my_max_affinity; + using proportional_mode::self; + linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()), + my_max_affinity(self().my_divisor) {} + linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} + linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} void set_affinity( task &t ) { - if( my_divisor ) + if( self().my_divisor ) t.set_affinity( affinity_id(my_head) + 1 ); } }; -//! Class determines whether template parameter has static boolean constant -//! 'is_splittable_in_proportion' initialized with value of 'true' or not. -/** If template parameter has such field that has been initialized with non-zero - * value then class field will be set to 'true', otherwise - 'false' - */ -template -class is_splittable_in_proportion { -private: - typedef char yes[1]; - typedef char no [2]; - - template static yes& decide(typename enable_if::type *); - template static no& decide(...); -public: - // equals to 'true' if and only if static const variable 'is_splittable_in_proportion' of template parameter - // initialized with the value of 'true' - static const bool value = (sizeof(decide(0)) == sizeof(yes)); -}; - -//! Provides default methods for non-balancing partition objects. -template -struct unbalancing_partition_type : Mode { - using Mode::self; - unbalancing_partition_type() : Mode() {} - unbalancing_partition_type(unbalancing_partition_type& p, split) : Mode(p, split()) {} - unbalancing_partition_type(unbalancing_partition_type& p, const proportional_split& split_obj) : Mode(p, split_obj) {} -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - template - proportional_split get_split() { - if (is_splittable_in_proportion::value) { - size_t size = self().my_divisor / Mode::my_partition::factor; -#if __TBB_NONUNIFORM_TASK_CREATION - size_t right = (size + 2) / 3; -#else - size_t right = size / 2; -#endif - size_t left = size - right; - return proportional_split(left, right); - } else { - return proportional_split(1, 1); - } - } -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back -}; - /*! Determine work-balance phase implementing splitting & stealing actions */ template -struct balancing_partition_type : unbalancing_partition_type { +struct dynamic_grainsize_mode : Mode { using Mode::self; #ifdef __TBB_USE_MACHINE_TIME_STAMPS tbb::internal::machine_tsc_t my_dst_tsc; @@ -379,27 +370,27 @@ struct balancing_partition_type : unbalancing_partition_type { } my_delay; depth_t my_max_depth; static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY; - balancing_partition_type(): unbalancing_partition_type() + dynamic_grainsize_mode(): Mode() #ifdef __TBB_USE_MACHINE_TIME_STAMPS , my_dst_tsc(0) #endif , my_delay(begin) , my_max_depth(__TBB_INIT_DEPTH) {} - balancing_partition_type(balancing_partition_type& p, split) - : unbalancing_partition_type(p, split()) + dynamic_grainsize_mode(dynamic_grainsize_mode& p, split) + : Mode(p, split()) #ifdef __TBB_USE_MACHINE_TIME_STAMPS , my_dst_tsc(0) #endif , my_delay(pass) , my_max_depth(p.my_max_depth) {} - balancing_partition_type(balancing_partition_type& p, const proportional_split& split_obj) - : unbalancing_partition_type(p, split_obj) + dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj) + : Mode(p, split_obj) #ifdef __TBB_USE_MACHINE_TIME_STAMPS , my_dst_tsc(0) #endif , my_delay(begin) , my_max_depth(p.my_max_depth) {} - bool check_being_stolen( task &t) { // part of old should_execute_range() + bool check_being_stolen(task &t) { // part of old should_execute_range() if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? if( t.is_stolen_task() && t.parent()->ref_count() >= 2 ) { // runs concurrently with the left task @@ -478,14 +469,14 @@ struct balancing_partition_type : unbalancing_partition_type { } }; -class auto_partition_type: public balancing_partition_type > { +class auto_partition_type: public dynamic_grainsize_mode > { public: auto_partition_type( const auto_partitioner& ) - : balancing_partition_type >() { + : dynamic_grainsize_mode >() { my_divisor *= __TBB_INITIAL_CHUNKS; } auto_partition_type( auto_partition_type& src, split) - : balancing_partition_type >(src, split()) {} + : dynamic_grainsize_mode >(src, split()) {} bool is_divisible() { // part of old should_execute_range() if( my_divisor > 1 ) return true; if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead @@ -517,25 +508,25 @@ class simple_partition_type: public partition_type_base { } }; -class static_partition_type : public unbalancing_partition_type > { +class static_partition_type : public linear_affinity_mode { public: typedef proportional_split split_type; static_partition_type( const static_partitioner& ) - : unbalancing_partition_type >() {} + : linear_affinity_mode() {} static_partition_type( static_partition_type& p, split ) - : unbalancing_partition_type >(p, split()) {} + : linear_affinity_mode(p, split()) {} static_partition_type( static_partition_type& p, const proportional_split& split_obj ) - : unbalancing_partition_type >(p, split_obj) {} + : linear_affinity_mode(p, split_obj) {} }; -class affinity_partition_type : public balancing_partition_type > { +class affinity_partition_type : public dynamic_grainsize_mode > { static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units tbb::internal::affinity_id* my_array; public: static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task typedef proportional_split split_type; affinity_partition_type( tbb::internal::affinity_partitioner_base_v3& ap ) - : balancing_partition_type >() { + : dynamic_grainsize_mode >() { __TBB_ASSERT( (factor&(factor-1))==0, "factor must be power of two" ); ap.resize(factor); my_array = ap.my_array; @@ -543,10 +534,10 @@ class affinity_partition_type : public balancing_partition_type >(p, split()) + : dynamic_grainsize_mode >(p, split()) , my_array(p.my_array) {} affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj) - : balancing_partition_type >(p, split_obj) + : dynamic_grainsize_mode >(p, split_obj) , my_array(p.my_array) {} void set_affinity( task &t ) { if( my_divisor ) { @@ -596,6 +587,7 @@ class simple_partitioner { template friend class serial::interface9::start_for; template friend class interface9::internal::start_for; template friend class interface9::internal::start_reduce; + template friend class interface9::internal::start_deterministic_reduce; template friend class internal::start_scan; // backward compatibility class partition_type: public internal::partition_type_base { @@ -641,6 +633,7 @@ class static_partitioner { template friend class serial::interface9::start_for; template friend class interface9::internal::start_for; template friend class interface9::internal::start_reduce; + template friend class interface9::internal::start_deterministic_reduce; template friend class internal::start_scan; // backward compatibility typedef interface9::internal::old_auto_partition_type partition_type; diff --git a/inst/include/tbb_local/tbb/pipeline.h b/inst/include/tbb_local/tbb/pipeline.h index 20a8ec9b..c7138644 100644 --- a/inst/include/tbb_local/tbb/pipeline.h +++ b/inst/include/tbb_local/tbb/pipeline.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_pipeline_H diff --git a/inst/include/tbb_local/tbb/queuing_mutex.h b/inst/include/tbb_local/tbb/queuing_mutex.h index 0fe4b3ea..15f94774 100644 --- a/inst/include/tbb_local/tbb/queuing_mutex.h +++ b/inst/include/tbb_local/tbb/queuing_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,29 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_queuing_mutex_H #define __TBB_queuing_mutex_H -#include "tbb_config.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - #include "atomic.h" #include "tbb_profiling.h" @@ -59,6 +42,7 @@ class queuing_mutex : internal::mutex_copy_deprecated_and_disabled { //! Initialize fields to mean "no lock held". void initialize() { mutex = NULL; + going = 0; #if TBB_USE_ASSERT internal::poison_pointer(next); #endif /* TBB_USE_ASSERT */ diff --git a/inst/include/tbb_local/tbb/queuing_rw_mutex.h b/inst/include/tbb_local/tbb/queuing_rw_mutex.h index e0224ed5..6d5d3130 100644 --- a/inst/include/tbb_local/tbb/queuing_rw_mutex.h +++ b/inst/include/tbb_local/tbb/queuing_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,29 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_queuing_rw_mutex_H #define __TBB_queuing_rw_mutex_H -#include "tbb_config.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - #include "atomic.h" #include "tbb_profiling.h" @@ -68,6 +51,8 @@ class queuing_rw_mutex : internal::mutex_copy_deprecated_and_disabled { //! Initialize fields to mean "no lock held". void initialize() { my_mutex = NULL; + my_internal_lock = 0; + my_going = 0; #if TBB_USE_ASSERT my_state = 0xFF; // Set to invalid state internal::poison_pointer(my_next); diff --git a/inst/include/tbb_local/tbb/reader_writer_lock.h b/inst/include/tbb_local/tbb/reader_writer_lock.h index 353beec5..2275abc7 100644 --- a/inst/include/tbb_local/tbb/reader_writer_lock.h +++ b/inst/include/tbb_local/tbb/reader_writer_lock.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_reader_writer_lock_H diff --git a/inst/include/tbb_local/tbb/recursive_mutex.h b/inst/include/tbb_local/tbb/recursive_mutex.h index 5a23c097..576bd999 100644 --- a/inst/include/tbb_local/tbb/recursive_mutex.h +++ b/inst/include/tbb_local/tbb/recursive_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_recursive_mutex_H diff --git a/inst/include/tbb_local/tbb/runtime_loader.h b/inst/include/tbb_local/tbb/runtime_loader.h index df284645..a6d695b3 100644 --- a/inst/include/tbb_local/tbb/runtime_loader.h +++ b/inst/include/tbb_local/tbb/runtime_loader.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_runtime_loader_H diff --git a/inst/include/tbb_local/tbb/scalable_allocator.h b/inst/include/tbb_local/tbb/scalable_allocator.h index c2a81493..a077e00c 100644 --- a/inst/include/tbb_local/tbb/scalable_allocator.h +++ b/inst/include/tbb_local/tbb/scalable_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_scalable_allocator_H @@ -95,9 +91,12 @@ typedef enum { TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ /* deprecated, kept for backward compatibility only */ USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value Bytes, clean internal buffers + /* try to limit memory consumption value (Bytes), clean internal buffers if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT + TBBMALLOC_SET_SOFT_HEAP_LIMIT, + /* Lower bound for the size (Bytes), that is interpreted as huge + * and not released during regular cleanup operations. */ + TBBMALLOC_SET_HUGE_SIZE_THRESHOLD } AllocationModeParam; /** Set TBB allocator-specific allocation modes. @@ -193,7 +192,9 @@ void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t ali bool pool_reset(MemoryPool* memPool); bool pool_free(MemoryPool *memPool, void *object); MemoryPool *pool_identify(void *object); -} +size_t pool_msize(MemoryPool *memPool, void *object); + +} // namespace rml #include /* To use new with the placement argument */ @@ -207,7 +208,11 @@ MemoryPool *pool_identify(void *object); #endif #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward +#include // std::forward +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include #endif namespace tbb { @@ -314,6 +319,48 @@ inline bool operator==( const scalable_allocator&, const scalable_allocator inline bool operator!=( const scalable_allocator&, const scalable_allocator& ) {return false;} +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +namespace internal { + +//! C++17 memory resource implementation for scalable allocator +//! ISO C++ Section 23.12.2 +class scalable_resource_impl : public std::pmr::memory_resource { +private: + void* do_allocate(size_t bytes, size_t alignment) override { + void* ptr = scalable_aligned_malloc( bytes, alignment ); + if (!ptr) { + throw_exception(std::bad_alloc()); + } + return ptr; + } + + void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override { + scalable_free(ptr); + } + + //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any + //! other instance of this class + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + return this == &other || +#if __TBB_USE_OPTIONAL_RTTI + dynamic_cast(&other) != NULL; +#else + false; +#endif + } +}; + +} // namespace internal + +//! Global scalable allocator memory resource provider +inline std::pmr::memory_resource* scalable_memory_resource() noexcept { + static tbb::internal::scalable_resource_impl scalable_res; + return &scalable_res; +} + +#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ + } // namespace tbb #if _MSC_VER diff --git a/inst/include/tbb_local/tbb/spin_mutex.h b/inst/include/tbb_local/tbb/spin_mutex.h index 99ef15c2..e29cf354 100644 --- a/inst/include/tbb_local/tbb/spin_mutex.h +++ b/inst/include/tbb_local/tbb/spin_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_spin_mutex_H diff --git a/inst/include/tbb_local/tbb/spin_rw_mutex.h b/inst/include/tbb_local/tbb/spin_rw_mutex.h index b20f4ebd..1477e684 100644 --- a/inst/include/tbb_local/tbb/spin_rw_mutex.h +++ b/inst/include/tbb_local/tbb/spin_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_spin_rw_mutex_H @@ -90,9 +86,6 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { class scoped_lock : internal::no_copy { #if __TBB_TSX_AVAILABLE friend class tbb::interface8::internal::x86_rtm_rw_mutex; - // helper methods for x86_rtm_rw_mutex - spin_rw_mutex *internal_get_mutex() const { return mutex; } - void internal_set_mutex(spin_rw_mutex* m) { mutex = m; } #endif public: //! Construct lock that has not acquired a mutex. @@ -121,15 +114,15 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { //! Upgrade reader to become a writer. /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( !is_writer, "not a reader" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); + if (is_writer) return true; // Already a writer is_writer = true; return mutex->internal_upgrade(); } //! Release lock. void release() { - __TBB_ASSERT( mutex, "lock is not acquired" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); spin_rw_mutex *m = mutex; mutex = NULL; #if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT @@ -143,8 +136,8 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { //! Downgrade writer to become a reader. bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( is_writer, "not a writer" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); + if (!is_writer) return true; // Already a reader #if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT mutex->internal_downgrade(); #else diff --git a/inst/include/tbb_local/tbb/task.h b/inst/include/tbb_local/tbb/task.h index 246684ab..6eff290f 100644 --- a/inst/include/tbb_local/tbb/task.h +++ b/inst/include/tbb_local/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_H @@ -49,7 +45,9 @@ namespace internal { //< @cond INTERNAL task* self; task& parent; public: - explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) {} + explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) { + suppress_unused_warning( self ); + } task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; void __TBB_EXPORTED_METHOD free( task& ) const; }; @@ -162,18 +160,25 @@ namespace internal { void __TBB_EXPORTED_METHOD free( task& ) const; }; +#if __TBB_PREVIEW_CRITICAL_TASKS + // TODO: move to class methods when critical task API becomes public + void make_critical( task& t ); + bool is_critical( task& t ); +#endif + //! Memory prefix to a task object. /** This class is internal to the library. Do not reference it directly, except within the library itself. - Fields are ordered in way that preserves backwards compatibility and yields - good packing on typical 32-bit and 64-bit platforms. New fields should be - added at the beginning for backward compatibility with accesses to the task - prefix inlined into application code. + Fields are ordered in way that preserves backwards compatibility and yields good packing on + typical 32-bit and 64-bit platforms. New fields should be added at the beginning for + backward compatibility with accesses to the task prefix inlined into application code. To + prevent ODR violation, the class shall have the same layout in all application translation + units. If some fields are conditional (e.g. enabled by preview macros) and might get + skipped, use reserved fields to adjust the layout. - In case task prefix size exceeds 32 or 64 bytes on IA32 and Intel64 - architectures correspondingly, consider dynamic setting of task_alignment - and task_prefix_reservation_size based on the maximal operand size supported - by the current CPU. + In case task prefix size exceeds 32 or 64 bytes on IA32 and Intel64 architectures + correspondingly, consider dynamic setting of task_alignment and task_prefix_reservation_size + based on the maximal operand size supported by the current CPU. @ingroup task_scheduling */ class task_prefix { @@ -186,10 +191,16 @@ namespace internal { friend class internal::allocate_child_proxy; friend class internal::allocate_continuation_proxy; friend class internal::allocate_additional_child_of_proxy; +#if __TBB_PREVIEW_CRITICAL_TASKS + friend void make_critical( task& ); + friend bool is_critical( task& ); +#endif #if __TBB_TASK_ISOLATION //! The tag used for task isolation. isolation_tag isolation; +#else + intptr_t reserved_space_for_task_isolation_tag; #endif /* __TBB_TASK_ISOLATION */ #if __TBB_TASK_GROUP_CONTEXT @@ -271,6 +282,10 @@ namespace internal { #if __TBB_TASK_PRIORITY namespace internal { static const int priority_stride_v4 = INT_MAX / 4; +#if __TBB_PREVIEW_CRITICAL_TASKS + // TODO: move into priority_t enum when critical tasks become public feature + static const int priority_critical = priority_stride_v4 * 3 + priority_stride_v4 / 3 * 2; +#endif } enum priority_t { @@ -291,6 +306,7 @@ enum priority_t { class task_scheduler_init; namespace interface7 { class task_arena; } +using interface7::task_arena; //! Used to form groups of tasks /** @ingroup task_scheduling @@ -317,7 +333,7 @@ class task_group_context : internal::no_copy { private: friend class internal::generic_scheduler; friend class task_scheduler_init; - friend class interface7::task_arena; + friend class task_arena; #if TBB_USE_CAPTURED_EXCEPTION typedef tbb_exception exception_container_type; @@ -417,12 +433,16 @@ class task_group_context : internal::no_copy { intptr_t my_priority; #endif /* __TBB_TASK_PRIORITY */ + //! Description of algorithm for scheduler based instrumentation. + internal::string_index my_name; + //! Trailing padding protecting accesses to frequently used members from false sharing /** \sa _leading_padding **/ char _trailing_padding[internal::NFS_MaxLineSize - 2 * sizeof(uintptr_t) - 2 * sizeof(void*) #if __TBB_TASK_PRIORITY - - sizeof(intptr_t) + - sizeof(intptr_t) #endif /* __TBB_TASK_PRIORITY */ + - sizeof(internal::string_index) ]; public: @@ -458,7 +478,17 @@ class task_group_context : internal::no_copy { task_group_context ( kind_type relation_with_parent = bound, uintptr_t t = default_traits ) : my_kind(relation_with_parent) - , my_version_and_traits(2 | t) + , my_version_and_traits(3 | t) + , my_name(internal::CUSTOM_CTX) + { + init(); + } + + // Custom constructor for instrumentation of tbb algorithm + task_group_context ( internal::string_index name ) + : my_kind(bound) + , my_version_and_traits(3 | default_traits) + , my_name(name) { init(); } @@ -780,11 +810,24 @@ class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { #if __TBB_TASK_PRIORITY //! Enqueue task for starvation-resistant execution on the specified priority level. static void enqueue( task& t, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); +#else + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); +#endif t.prefix().owner->enqueue( t, (void*)p ); } #endif /* __TBB_TASK_PRIORITY */ + //! Enqueue task in task_arena + //! The implementation is in task_arena.h + inline static void enqueue( task& t, task_arena& arena +#if __TBB_TASK_PRIORITY + , priority_t p = priority_t(0) +#endif + ); + //! The innermost task being executed or destroyed by the current thread at the moment. static task& __TBB_EXPORTED_FUNC self(); @@ -903,8 +946,19 @@ class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { internal::task_prefix& prefix( internal::version_tag* = NULL ) const { return reinterpret_cast(const_cast(this))[-1]; } +#if __TBB_PREVIEW_CRITICAL_TASKS + friend void internal::make_critical( task& ); + friend bool internal::is_critical( task& ); +#endif }; // class task +#if __TBB_PREVIEW_CRITICAL_TASKS +namespace internal { +inline void make_critical( task& t ) { t.prefix().extra_state |= 0x8; } +inline bool is_critical( task& t ) { return bool((t.prefix().extra_state & 0x8) != 0); } +} // namespace internal +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ + //! task that does nothing. Useful for synchronization. /** @ingroup task_scheduling */ class empty_task: public task { @@ -928,6 +982,9 @@ namespace internal { } public: function_task( const F& f ) : my_func(f) {} +#if __TBB_CPP11_RVALUE_REF_PRESENT + function_task( F&& f ) : my_func( std::move(f) ) {} +#endif }; } // namespace internal //! @endcond @@ -948,7 +1005,7 @@ class task_list: internal::no_copy { //! Destroys the list, but does not destroy the task objects. ~task_list() {} - //! True if list if empty; false otherwise. + //! True if list is empty; false otherwise. bool empty() const {return !first;} //! Push task onto back of list. diff --git a/inst/include/tbb_local/tbb/task_arena.h b/inst/include/tbb_local/tbb/task_arena.h index f33135b6..f1fef56e 100644 --- a/inst/include/tbb_local/tbb/task_arena.h +++ b/inst/include/tbb_local/tbb/task_arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_arena_H @@ -23,9 +19,11 @@ #include "task.h" #include "tbb_exception.h" +#include "internal/_template_helpers.h" #if TBB_USE_THREADING_TOOLS #include "atomic.h" // for as_atomic #endif +#include "aligned_space.h" namespace tbb { @@ -55,14 +53,47 @@ class delegate_base : no_assign { virtual ~delegate_base() {} }; -template +// If decltype is available, the helper detects the return type of functor of specified type, +// otherwise it defines the void type. +template +struct return_type_or_void { +#if __TBB_CPP11_DECLTYPE_PRESENT && !__TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN + typedef decltype(declval()()) type; +#else + typedef void type; +#endif +}; + +template class delegated_function : public delegate_base { + F &my_func; + tbb::aligned_space my_return_storage; + // The function should be called only once. + void operator()() const __TBB_override { + new (my_return_storage.begin()) R(my_func()); + } +public: + delegated_function(F& f) : my_func(f) {} + // The function can be called only after operator() and only once. + R consume_result() const { + return tbb::internal::move(*(my_return_storage.begin())); + } + ~delegated_function() { + my_return_storage.begin()->~R(); + } +}; + +template +class delegated_function : public delegate_base { F &my_func; void operator()() const __TBB_override { my_func(); } public: - delegated_function ( F& f ) : my_func(f) {} + delegated_function(F& f) : my_func(f) {} + void consume_result() const {} + + friend class task_arena_base; }; class task_arena_base { @@ -119,6 +150,13 @@ class task_arena_base { #if __TBB_TASK_ISOLATION void __TBB_EXPORTED_FUNC isolate_within_arena( delegate_base& d, intptr_t reserved = 0 ); + +template +R isolate_impl(F& f) { + delegated_function d(f); + isolate_within_arena(d); + return d.consume_result(); +} #endif /* __TBB_TASK_ISOLATION */ } // namespace internal //! @endcond @@ -130,6 +168,11 @@ void __TBB_EXPORTED_FUNC isolate_within_arena( delegate_base& d, intptr_t reserv */ class task_arena : public internal::task_arena_base { friend class tbb::internal::task_scheduler_observer_v3; + friend void task::enqueue(task&, task_arena& +#if __TBB_TASK_PRIORITY + , priority_t +#endif + ); friend int tbb::this_task_arena::max_concurrency(); bool my_initialized; void mark_initialized() { @@ -146,6 +189,31 @@ class task_arena : public internal::task_arena_base { #endif } + template + void enqueue_impl( __TBB_FORWARDING_REF(F) f +#if __TBB_TASK_PRIORITY + , priority_t p = priority_t(0) +#endif + ) { +#if !__TBB_TASK_PRIORITY + intptr_t p = 0; +#endif + initialize(); +#if __TBB_TASK_GROUP_CONTEXT + internal_enqueue(*new(task::allocate_root(*my_context)) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); +#else + internal_enqueue(*new(task::allocate_root()) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); +#endif /* __TBB_TASK_GROUP_CONTEXT */ + } + + template + R execute_impl(F& f) { + initialize(); + internal::delegated_function d(f); + internal_execute(d); + return d.consume_result(); + } + public: //! Creates task_arena with certain concurrency limits /** Sets up settings only, real construction is deferred till the first method invocation @@ -187,7 +255,7 @@ class task_arena : public internal::task_arena_base { //! Overrides concurrency level and forces initialization of internal representation inline void initialize(int max_concurrency_, unsigned reserved_for_masters = 1) { // TODO: decide if this call must be thread-safe - __TBB_ASSERT( !my_arena, "Impossible to modify settings of an already initialized task_arena"); + __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); if( !my_initialized ) { my_max_concurrency = max_concurrency_; my_master_slots = reserved_for_masters; @@ -198,10 +266,10 @@ class task_arena : public internal::task_arena_base { //! Attaches this instance to the current arena of the thread inline void initialize(attach) { // TODO: decide if this call must be thread-safe - __TBB_ASSERT( !my_arena, "Impossible to modify settings of an already initialized task_arena"); + __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); if( !my_initialized ) { internal_attach(); - if( !my_arena ) internal_initialize(); + if ( !my_arena ) internal_initialize(); mark_initialized(); } } @@ -227,49 +295,62 @@ class task_arena : public internal::task_arena_base { //! Enqueues a task into the arena to process a functor, and immediately returns. //! Does not require the calling thread to join the arena + +#if __TBB_CPP11_RVALUE_REF_PRESENT template - void enqueue( const F& f ) { - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue( *new( task::allocate_root(*my_context) ) internal::function_task(f), 0 ); + void enqueue( F&& f ) { + enqueue_impl(std::forward(f)); + } #else - internal_enqueue( *new( task::allocate_root() ) internal::function_task(f), 0 ); -#endif + template + void enqueue( const F& f ) { + enqueue_impl(f); } +#endif #if __TBB_TASK_PRIORITY //! Enqueues a task with priority p into the arena to process a functor f, and immediately returns. //! Does not require the calling thread to join the arena template +#if __TBB_CPP11_RVALUE_REF_PRESENT + void enqueue( F&& f, priority_t p ) { +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); +#else + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); +#endif + enqueue_impl(std::forward(f), p); + } +#else void enqueue( const F& f, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue( *new( task::allocate_root(*my_context) ) internal::function_task(f), (intptr_t)p ); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); #else - internal_enqueue( *new( task::allocate_root() ) internal::function_task(f), (intptr_t)p ); + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); #endif + enqueue_impl(f,p); } +#endif #endif// __TBB_TASK_PRIORITY - //! Joins the arena and executes a functor, then returns + //! Joins the arena and executes a mutable functor, then returns //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). template - void execute(F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute( d ); + typename internal::return_type_or_void::type execute(F& f) { + return execute_impl::type>(f); } - //! Joins the arena and executes a functor, then returns + //! Joins the arena and executes a constant functor, then returns //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). template - void execute(const F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute( d ); + typename internal::return_type_or_void::type execute(const F& f) { + return execute_impl::type>(f); } #if __TBB_EXTRA_DEBUG @@ -297,14 +378,21 @@ class task_arena : public internal::task_arena_base { #if __TBB_TASK_ISOLATION namespace this_task_arena { + //! Executes a mutable functor in isolation within the current task arena. + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). template - void isolate( const F& f ) { - internal::delegated_function d(f); - internal::isolate_within_arena( d ); + typename internal::return_type_or_void::type isolate(F& f) { + return internal::isolate_impl::type>(f); + } + + //! Executes a constant functor in isolation within the current task arena. + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). + template + typename internal::return_type_or_void::type isolate(const F& f) { + return internal::isolate_impl::type>(f); } } #endif /* __TBB_TASK_ISOLATION */ - } // namespace interfaceX using interface7::task_arena; @@ -325,9 +413,21 @@ namespace this_task_arena { inline int max_concurrency() { return tbb::task_arena::internal_max_concurrency(NULL); } - } // namespace this_task_arena +//! Enqueue task in task_arena +void task::enqueue( task& t, task_arena& arena +#if __TBB_TASK_PRIORITY + , priority_t p +#endif + ) { +#if !__TBB_TASK_PRIORITY + intptr_t p = 0; +#endif + arena.initialize(); + //! Note: the context of the task may differ from the context instantiated by task_arena + arena.internal_enqueue(t, p); +} } // namespace tbb #endif /* __TBB_task_arena_H */ diff --git a/inst/include/tbb_local/tbb/task_group.h b/inst/include/tbb_local/tbb/task_group.h index bf6922b9..cafde9f2 100644 --- a/inst/include/tbb_local/tbb/task_group.h +++ b/inst/include/tbb_local/tbb/task_group.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_group_H @@ -23,6 +19,7 @@ #include "task.h" #include "tbb_exception.h" +#include "internal/_template_helpers.h" #if __TBB_TASK_GROUP_CONTEXT @@ -54,6 +51,9 @@ class task_handle : internal::no_assign { } public: task_handle( const F& f ) : my_func(f), my_state(0) {} +#if __TBB_CPP11_RVALUE_REF_PRESENT + task_handle( F&& f ) : my_func( std::move(f)), my_state(0) {} +#endif void operator() () const { my_func(); } }; @@ -78,6 +78,16 @@ class task_handle_task : public task { }; class task_group_base : internal::no_copy { + class ref_count_guard : internal::no_copy { + task& my_task; + public: + ref_count_guard(task& t) : my_task(t) { + my_task.increment_ref_count(); + } + ~ref_count_guard() { + my_task.decrement_ref_count(); + } + }; protected: empty_task* my_root; task_group_context my_context; @@ -87,17 +97,21 @@ class task_group_base : internal::no_copy { template task_group_status internal_run_and_wait( F& f ) { __TBB_TRY { - if ( !my_context.is_group_execution_cancelled() ) + if ( !my_context.is_group_execution_cancelled() ) { + // We need to increase the reference count of the root task to notify waiters that + // this task group has some work in progress. + ref_count_guard guard(*my_root); f(); + } } __TBB_CATCH( ... ) { my_context.register_pending_exception(); } return wait(); } - template - void internal_run( F& f ) { - owner().spawn( *new( owner().allocate_additional_child_of(*my_root) ) Task(f) ); + template + void internal_run( __TBB_FORWARDING_REF(F) f ) { + owner().spawn( *new( owner().allocate_additional_child_of(*my_root) ) Task( internal::forward(f) )); } public: @@ -110,7 +124,11 @@ class task_group_base : internal::no_copy { ~task_group_base() __TBB_NOEXCEPT(false) { if( my_root->ref_count() > 1 ) { +#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT + bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0; +#else bool stack_unwinding_in_progress = std::uncaught_exception(); +#endif // Always attempt to do proper cleanup to avoid inevitable memory corruption // in case of missing wait (for the sake of better testability & debuggability) if ( !is_canceling() ) @@ -132,7 +150,7 @@ class task_group_base : internal::no_copy { template void run( task_handle& h ) { - internal_run< task_handle, internal::task_handle_task >( h ); + internal_run< internal::task_handle_task >( h ); } task_group_status wait() { @@ -168,22 +186,30 @@ class task_group : public internal::task_group_base { #if __SUNPRO_CC template void run( task_handle& h ) { - internal_run< task_handle, internal::task_handle_task >( h ); + internal_run< internal::task_handle_task >( h ); } #else using task_group_base::run; #endif +#if __TBB_CPP11_RVALUE_REF_PRESENT + template + void run( F&& f ) { + internal_run< internal::function_task< typename internal::strip::type > >( std::forward< F >(f) ); + } +#else template - void run( const F& f ) { - internal_run< const F, internal::function_task >( f ); + void run(const F& f) { + internal_run >(f); } +#endif template task_group_status run_and_wait( const F& f ) { return internal_run_and_wait( f ); } + // TODO: add task_handle rvalues support template task_group_status run_and_wait( task_handle& h ) { h.mark_scheduled(); @@ -193,6 +219,7 @@ class task_group : public internal::task_group_base { class structured_task_group : public internal::task_group_base { public: + // TODO: add task_handle rvalues support template task_group_status run_and_wait ( task_handle& h ) { h.mark_scheduled(); @@ -211,10 +238,17 @@ bool is_current_task_group_canceling() { return task::self().is_cancelled(); } +#if __TBB_CPP11_RVALUE_REF_PRESENT +template +task_handle< typename internal::strip::type > make_task( F&& f ) { + return task_handle< typename internal::strip::type >( std::forward(f) ); +} +#else template task_handle make_task( const F& f ) { return task_handle( f ); } +#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ } // namespace tbb diff --git a/inst/include/tbb_local/tbb/task_scheduler_init.h b/inst/include/tbb_local/tbb/task_scheduler_init.h index 928e7a4e..66a24086 100644 --- a/inst/include/tbb_local/tbb/task_scheduler_init.h +++ b/inst/include/tbb_local/tbb/task_scheduler_init.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_scheduler_init_H diff --git a/inst/include/tbb_local/tbb/task_scheduler_observer.h b/inst/include/tbb_local/tbb/task_scheduler_observer.h index 5586ad4f..72694af3 100644 --- a/inst/include/tbb_local/tbb/task_scheduler_observer.h +++ b/inst/include/tbb_local/tbb/task_scheduler_observer.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,17 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_scheduler_observer_H #define __TBB_task_scheduler_observer_H #include "atomic.h" -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER #include "task_arena.h" #endif @@ -88,7 +84,7 @@ class task_scheduler_observer_v3 { } // namespace internal -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER namespace interface6 { class task_scheduler_observer : public internal::task_scheduler_observer_v3 { friend class internal::task_scheduler_observer_v3; @@ -149,23 +145,13 @@ class task_scheduler_observer : public internal::task_scheduler_observer_v3 { } internal::task_scheduler_observer_v3::observe(state); } - -#if __TBB_SLEEP_PERMISSION - //! Return commands for may_sleep() - enum { keep_awake = false, allow_sleep = true }; - - //! The callback can be invoked by a worker thread before it goes to sleep. - /** If it returns false ('keep_awake'), the thread will keep spinning and looking for work. - It will not be called for master threads. **/ - virtual bool may_sleep() { return allow_sleep; } -#endif /*__TBB_SLEEP_PERMISSION*/ }; } //namespace interface6 using interface6::task_scheduler_observer; -#else /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ +#else /*__TBB_ARENA_OBSERVER*/ typedef tbb::internal::task_scheduler_observer_v3 task_scheduler_observer; -#endif /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ +#endif /*__TBB_ARENA_OBSERVER*/ } // namespace tbb diff --git a/inst/include/tbb_local/tbb/tbb.h b/inst/include/tbb_local/tbb/tbb.h index 5e385ea2..ba4b1122 100644 --- a/inst/include/tbb_local/tbb/tbb.h +++ b/inst/include/tbb_local/tbb/tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_H @@ -23,7 +19,7 @@ /** This header bulk-includes declarations or definitions of all the functionality - provided by TBB (save for malloc dependent headers). + provided by TBB (save for tbbmalloc and 3rd party dependent headers). If you use only a few TBB constructs, consider including specific headers only. Any header listed below can be included independently of others. @@ -37,6 +33,9 @@ #include "blocked_range.h" #include "blocked_range2d.h" #include "blocked_range3d.h" +#if TBB_PREVIEW_BLOCKED_RANGE_ND +#include "blocked_rangeNd.h" +#endif #include "cache_aligned_allocator.h" #include "combinable.h" #include "concurrent_hash_map.h" @@ -47,13 +46,16 @@ #include "concurrent_queue.h" #include "concurrent_unordered_map.h" #include "concurrent_unordered_set.h" +#if TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#include "concurrent_map.h" +#include "concurrent_set.h" +#endif #include "concurrent_vector.h" #include "critical_section.h" #include "enumerable_thread_specific.h" #include "flow_graph.h" -#if TBB_PREVIEW_GLOBAL_CONTROL #include "global_control.h" -#endif +#include "iterators.h" #include "mutex.h" #include "null_mutex.h" #include "null_rw_mutex.h" diff --git a/inst/include/tbb_local/tbb/tbb_allocator.h b/inst/include/tbb_local/tbb/tbb_allocator.h index a3f1ef0c..20659387 100644 --- a/inst/include/tbb_local/tbb/tbb_allocator.h +++ b/inst/include/tbb_local/tbb/tbb_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_allocator_H @@ -26,19 +22,8 @@ #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC #include // std::forward #endif - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - namespace tbb { //! @cond INTERNAL @@ -185,7 +170,7 @@ class zero_allocator : public Allocator pointer allocate(const size_type n, const void *hint = 0 ) { pointer ptr = base_allocator_type::allocate( n, hint ); - std::memset( (void*) ptr, 0, n * sizeof(value_type) ); + std::memset( static_cast(ptr), 0, n * sizeof(value_type) ); return ptr; } }; diff --git a/inst/include/tbb_local/tbb/tbb_config.h b/inst/include/tbb_local/tbb/tbb_config.h index bdc8dcb9..dbfc9fff 100644 --- a/inst/include/tbb_local/tbb/tbb_config.h +++ b/inst/include/tbb_local/tbb/tbb_config.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_config_H @@ -34,29 +30,32 @@ */ #define __TBB_TODO 0 -/*Check which standard library we use on macOS*.*/ -/*__TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed.*/ -#if !defined(__TBB_SYMBOL) && (__APPLE__ || __ANDROID__) +/* Check which standard library we use. */ +/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */ +#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY #include #endif -// note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match +// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match // the actual GCC version on the system. #define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -// Since GNU libstdc++ does not have a convenient macro for its version, -// we rely on the version of GCC or the user-specified macro below. -// The format of TBB_USE_GLIBCXX_VERSION should match the __TBB_GCC_VERSION above, -// e.g. it should be set to 40902 for libstdc++ coming with GCC 4.9.2. +// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro. +// Therefore we use different ways to detect its version. #ifdef TBB_USE_GLIBCXX_VERSION +// The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro. +// Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1. #define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION -#else +#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__ +// Reported versions of GCC and libstdc++ do not match; trust the latter +#define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000) +#elif __GLIBCPP__ || __GLIBCXX__ +// The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION #define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION -//TODO: analyze __GLIBCXX__ instead of __TBB_GCC_VERSION ? #endif #if __clang__ - /**according to clang documentation version can be vendor specific **/ + // according to clang documentation, version can be vendor specific #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) #endif @@ -65,6 +64,16 @@ #define __TBB_IOS 1 #endif +#if __APPLE__ + #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \ + && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000 + // ICC does not correctly set the macro if -mmacosx-min-version is not specified + #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000)) + #else + #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ + #endif +#endif + /** Preprocessor symbols to determine HW architecture **/ #if _WIN32||_WIN64 @@ -108,6 +117,14 @@ #define __INTEL_COMPILER 1210 #endif +#if __clang__ && !__INTEL_COMPILER +#define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) +#elif defined(_CPPRTTI) +#define __TBB_USE_OPTIONAL_RTTI 1 +#else +#define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) +#endif + #if __TBB_GCC_VERSION >= 40400 && !defined(__INTEL_COMPILER) /** warning suppression pragmas available in GCC since 4.4 **/ #define __TBB_GCC_WARNING_SUPPRESSION_PRESENT 1 @@ -124,7 +141,17 @@ support added. */ -/** C++11 mode detection macros for Intel(R) C++ compiler (enabled by -std=c++XY option): +/** + __TBB_CPP11_PRESENT macro indicates that the compiler supports vast majority of C++11 features. + Depending on the compiler, some features might still be unsupported or work incorrectly. + Use it when enabling C++11 features individually is not practical, and be aware that + some "good enough" compilers might be excluded. **/ +#define __TBB_CPP11_PRESENT (__cplusplus >= 201103L || _MSC_VER >= 1900) + +#define __TBB_CPP17_FALLTHROUGH_PRESENT (__cplusplus >= 201703L) +#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER) + +/** C++11 mode detection macros for Intel(R) C++ Compiler (enabled by -std=c++XY option): __INTEL_CXX11_MODE__ for version >=13.0 (not available for ICC 15.0 if -std=c++14 is used), __STDC_HOSTED__ for version >=12.0 (useful only on Windows), __GXX_EXPERIMENTAL_CXX0X__ for version >=12.0 on Linux and macOS. **/ @@ -133,16 +160,9 @@ #define __INTEL_CXX11_MODE__ (__GXX_EXPERIMENTAL_CXX0X__ || (_MSC_VER && __STDC_HOSTED__)) #endif -// Intel(R) C++ Compiler offloading API to the Intel(R) Graphics Technology presence macro -// TODO: add support for ICC 15.00 _GFX_enqueue API and then decrease Intel compiler supported version -// TODO: add linux support and restict it with (__linux__ && __TBB_x86_64 && !__ANDROID__) macro -#if __INTEL_COMPILER >= 1600 && _WIN32 -#define __TBB_GFX_PRESENT 1 -#endif - #if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MODE__) // On Windows, C++11 features supported by Visual Studio 2010 and higher are enabled by default, - // so in absence of /Qstd= use MSVC branch for __TBB_CPP11_* detection. + // so in absence of /Qstd= use MSVC branch for feature detection. // On other platforms, no -std= means C++03. #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (__INTEL_CXX11_MODE__ && __VARIADIC_TEMPLATES) @@ -169,14 +189,7 @@ #endif #define __TBB_STATIC_ASSERT_PRESENT (__INTEL_CXX11_MODE__ || _MSC_VER >= 1600) #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40300 || _LIBCPP_VERSION)) - #if (__clang__ && __INTEL_COMPILER > 1400) - /* Older versions of Intel Compiler do not have __has_include */ - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #else - #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) - #endif + #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) #define __TBB_CONSTEXPR_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) /** ICC seems to disable support of noexcept event in c++11 when compiling in compatibility mode for gcc <4.6 **/ @@ -189,10 +202,12 @@ #define __TBB_OVERRIDE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) #define __TBB_ALIGNAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1500) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 1910) // a future version + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) #elif __clang__ /** TODO: these options need to be rechecked **/ -/** on macOS the only way to get C++11 is to use clang. For library features (e.g. exception_ptr) libc++ is also - * required. So there is no need to check GCC version for clang**/ #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __has_feature(__cxx_variadic_templates__) #define __TBB_CPP11_RVALUE_REF_PRESENT (__has_feature(__cxx_rvalue_references__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40500)) #define __TBB_IMPLICIT_MOVE_PRESENT __has_feature(cxx_implicit_moves) @@ -219,6 +234,10 @@ #define __TBB_OVERRIDE_PRESENT __has_feature(cxx_override_control) #define __TBB_ALIGNAS_PRESENT __has_feature(cxx_alignas) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __has_feature(cxx_alias_templates) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__has_feature(__cpp_deduction_guides)) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__has_feature(__cpp_lib_is_invocable)) #elif __GNUC__ #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __GXX_EXPERIMENTAL_CXX0X__ #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) @@ -243,8 +262,13 @@ #define __TBB_OVERRIDE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) #define __TBB_ALIGNAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40800) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L && __TBB_GCC_VERSION >= 70000) #elif _MSC_VER - // These definitions are also used with Intel Compiler in "default" mode; see a comment above. + // These definitions are also used with Intel C++ Compiler in "default" mode (__INTEL_CXX11_MODE__ == 0); + // see a comment in "__INTEL_COMPILER" section above. #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (_MSC_VER >= 1800) // Contains a workaround for ICC 13 @@ -265,47 +289,72 @@ #define __TBB_OVERRIDE_PRESENT (_MSC_VER >= 1700) #define __TBB_ALIGNAS_PRESENT (_MSC_VER >= 1900) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (_MSC_VER >= 1800) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (_MSC_VER >= 1900) + /* Variable templates are supported in VS2015 Update 2 or later */ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1914) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1911) #else - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT 0 - #define __TBB_CPP11_RVALUE_REF_PRESENT 0 - #define __TBB_IMPLICIT_MOVE_PRESENT 0 - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #define __TBB_STATIC_ASSERT_PRESENT 0 - #define __TBB_CPP11_TUPLE_PRESENT 0 - #define __TBB_INITIALIZER_LISTS_PRESENT 0 - #define __TBB_CONSTEXPR_PRESENT 0 - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT 0 - #define __TBB_NOEXCEPT_PRESENT 0 - #define __TBB_CPP11_STD_BEGIN_END_PRESENT 0 - #define __TBB_CPP11_AUTO_PRESENT 0 - #define __TBB_CPP11_DECLTYPE_PRESENT 0 - #define __TBB_CPP11_LAMBDAS_PRESENT 0 - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT 0 - #define __TBB_OVERRIDE_PRESENT 0 - #define __TBB_ALIGNAS_PRESENT 0 - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT 0 + #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_RVALUE_REF_PRESENT __TBB_CPP11_PRESENT + #define __TBB_IMPLICIT_MOVE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_EXCEPTION_PTR_PRESENT __TBB_CPP11_PRESENT + #define __TBB_STATIC_ASSERT_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_TUPLE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_INITIALIZER_LISTS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CONSTEXPR_PRESENT __TBB_CPP11_PRESENT + #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT __TBB_CPP11_PRESENT + #define __TBB_NOEXCEPT_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_STD_BEGIN_END_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_AUTO_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_DECLTYPE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_LAMBDAS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_OVERRIDE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_ALIGNAS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cplusplus >= 201703L) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) #endif // C++11 standard library features +#define __TBB_CPP11_ARRAY_PRESENT (_MSC_VER >= 1700 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300) + #ifndef __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif -#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >=1800) +#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >= 1800) -#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && (__GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L))) -#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) +#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) +#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) // GCC supported some of type properties since 4.7 -#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) +#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) // In GCC, std::move_if_noexcept appeared later than noexcept -#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) -#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1700 || \ - __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) -#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - -#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && _GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) - +#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) +#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1800 || \ + __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) +#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || __SUNPRO_CC)) + +// Due to libc++ limitations in C++03 mode, do not pass rvalues to std::make_shared() +#define __TBB_CPP11_SMART_POINTERS_PRESENT ( _MSC_VER >= 1600 || _LIBCPP_VERSION \ + || ((__cplusplus >= 201103L || __GXX_EXPERIMENTAL_CXX0X__) \ + && (__TBB_GLIBCXX_VERSION >= 40500 || __TBB_GLIBCXX_VERSION >= 40400 && __TBB_USE_OPTIONAL_RTTI)) ) + +#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) + +#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) + +#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \ + || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200)) +// TODO: wait when memory_resource will be fully supported in clang and define the right macro +// Currently it is in experimental stage since 6 version. +#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (_MSVC_LANG > 201402L || __cplusplus > 201402L) || \ + __GLIBCXX__ && __cpp_lib_memory_resource >= 201603) +#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911) // std::swap is in only since C++11, though MSVC had it at least since VS2005 #if _MSC_VER>=1400 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ #define __TBB_STD_SWAP_HEADER @@ -340,11 +389,21 @@ #define __TBB_GCC_BUILTIN_ATOMICS_PRESENT 1 #endif +#if __TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER && !__clang__ + // After GCC7 there was possible reordering problem in generic atomic load/store operations. + // So always using builtins. + #define TBB_USE_GCC_BUILTINS 1 +#endif + #if __INTEL_COMPILER >= 1200 /** built-in C++11 style atomics available in ICC since 12.0 **/ #define __TBB_ICC_BUILTIN_ATOMICS_PRESENT 1 #endif +#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) + #define __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT 1 +#endif + #define __TBB_TSX_INTRINSICS_PRESENT ((__RTM__ || _MSC_VER>=1700 || __INTEL_COMPILER>=1300) && !__TBB_DEFINE_MIC && !__ANDROID__) /** Macro helpers **/ @@ -361,8 +420,8 @@ /* There are four cases that are supported: 1. "_DEBUG is undefined" means "no debug"; - 2. "_DEBUG defined to something that is evaluated to 0 (the "garbage" is also evaluated to 0 [cpp.cond])" means "no debug"; - 3. "_DEBUG defined to something that is evaluated to non-zero value" means "debug"; + 2. "_DEBUG defined to something that is evaluated to 0" (including "garbage", as per [cpp.cond]) means "no debug"; + 3. "_DEBUG defined to something that is evaluated to a non-zero value" means "debug"; 4. "_DEBUG defined to nothing (empty)" means "debug". */ #ifdef _DEBUG @@ -391,7 +450,7 @@ There are four cases that are supported: #define TBB_USE_PERFORMANCE_WARNINGS TBB_PERFORMANCE_WARNINGS #else #define TBB_USE_PERFORMANCE_WARNINGS TBB_USE_DEBUG -#endif /* TBB_PEFORMANCE_WARNINGS */ +#endif /* TBB_PERFORMANCE_WARNINGS */ #endif /* TBB_USE_PERFORMANCE_WARNINGS */ #if __TBB_DEFINE_MIC @@ -410,18 +469,10 @@ There are four cases that are supported: #define TBB_USE_EXCEPTIONS 1 #endif -#if __clang__ && !__INTEL_COMPILER -#define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) -#elif defined(_CPPRTTI) -#define __TBB_USE_OPTIONAL_RTTI 1 -#else -#define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) -#endif - #ifndef TBB_IMPLEMENT_CPP0X /** By default, use C++11 classes if available **/ #if __clang__ - /* Old versions of Intel Compiler do not have __has_include or cannot use it in #define */ + /* Old versions of Intel C++ Compiler do not have __has_include or cannot use it in #define */ #if (__INTEL_COMPILER && (__INTEL_COMPILER < 1500 || __INTEL_COMPILER == 1500 && __INTEL_COMPILER_UPDATE <= 1)) #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || !_LIBCPP_VERSION) #else @@ -458,6 +509,15 @@ There are four cases that are supported: /** Internal TBB features & modes **/ +/** __TBB_CONCURRENT_ORDERED_CONTAINERS indicates that all conditions of use + * concurrent_map and concurrent_set are met. **/ +// TODO: Add cpp11 random generation macro +#ifndef __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + #define __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT ( __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ + && __TBB_IMPLICIT_MOVE_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT && __TBB_CPP11_ARRAY_PRESENT \ + && __TBB_INITIALIZER_LISTS_PRESENT ) +#endif + /** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ #ifndef __TBB_WEAK_SYMBOLS_PRESENT #define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) @@ -499,29 +559,13 @@ There are four cases that are supported: #define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official #ifndef __TBB_ARENA_OBSERVER - #define __TBB_ARENA_OBSERVER ((__TBB_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) + #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER #endif /* __TBB_ARENA_OBSERVER */ -#ifndef __TBB_SLEEP_PERMISSION - #define __TBB_SLEEP_PERMISSION ((__TBB_CPF_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_SLEEP_PERMISSION */ - #ifndef __TBB_TASK_ISOLATION - #define __TBB_TASK_ISOLATION (__TBB_CPF_BUILD||TBB_PREVIEW_TASK_ISOLATION) + #define __TBB_TASK_ISOLATION 1 #endif /* __TBB_TASK_ISOLATION */ -#if TBB_PREVIEW_FLOW_GRAPH_TRACE -// Users of flow-graph trace need to explicitly link against the preview library. This -// prevents the linker from implicitly linking an application with a preview version of -// TBB and unexpectedly bringing in other community preview features, which might change -// the behavior of the application. -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif /* TBB_PREVIEW_FLOW_GRAPH_TRACE */ - -#ifndef __TBB_ITT_STRUCTURE_API -#define __TBB_ITT_STRUCTURE_API ( !__TBB_DEFINE_MIC && (__TBB_CPF_BUILD || TBB_PREVIEW_FLOW_GRAPH_TRACE) ) -#endif - #if TBB_USE_EXCEPTIONS && !__TBB_TASK_GROUP_CONTEXT #error TBB_USE_EXCEPTIONS requires __TBB_TASK_GROUP_CONTEXT to be enabled #endif @@ -548,14 +592,8 @@ There are four cases that are supported: #endif /* __TBB_SURVIVE_THREAD_SWITCH */ #ifndef __TBB_DEFAULT_PARTITIONER -#if TBB_DEPRECATED -/** Default partitioner for parallel loop templates in TBB 1.0-2.1 */ -#define __TBB_DEFAULT_PARTITIONER tbb::simple_partitioner -#else -/** Default partitioner for parallel loop templates since TBB 2.2 */ #define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner -#endif /* TBB_DEPRECATED */ -#endif /* !defined(__TBB_DEFAULT_PARTITIONER */ +#endif #ifndef __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES #define __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES 1 @@ -577,8 +615,9 @@ There are four cases that are supported: #endif #endif -/** __TBB_WIN8UI_SUPPORT enables support of New Windows*8 Store Apps and limit a possibility to load +/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load shared libraries at run time only from application container **/ +// TODO: Separate this single macro into two for Windows 8 Store* (win8ui mode) and UWP/UWD modes. #if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP #define __TBB_WIN8UI_SUPPORT 1 #else @@ -627,7 +666,7 @@ There are four cases that are supported: #endif #if (_WIN32||_WIN64) && __INTEL_COMPILER == 1110 - /** That's a bug in Intel(R) C++ Compiler 11.1.044/IA-32 architecture/Windows* OS, that leads to a worker thread crash on the thread's startup. **/ + /** That's a bug in Intel C++ Compiler 11.1.044/IA-32 architecture/Windows* OS, that leads to a worker thread crash on the thread's startup. **/ #define __TBB_ICL_11_1_CODE_GEN_BROKEN 1 #endif @@ -658,12 +697,12 @@ There are four cases that are supported: #endif /* __FreeBSD__ */ #if (__linux__ || __APPLE__) && __i386__ && defined(__INTEL_COMPILER) - /** The Intel(R) C++ Compiler for IA-32 architecture (Linux* OS|macOS) crashes or generates + /** The Intel C++ Compiler for IA-32 architecture (Linux* OS|macOS) crashes or generates incorrect code when __asm__ arguments have a cast to volatile. **/ #define __TBB_ICC_ASM_VOLATILE_BROKEN 1 #endif -#if !__INTEL_COMPILER && (_MSC_VER || __GNUC__==3 && __GNUC_MINOR__<=2) +#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1700 || __GNUC__==3 && __GNUC_MINOR__<=2) /** Bug in GCC 3.2 and MSVC compilers that sometimes return 0 for __alignof(T) when T has not yet been instantiated. **/ #define __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN 1 @@ -686,11 +725,7 @@ There are four cases that are supported: #endif #if __INTEL_COMPILER==1300 && __TBB_GLIBCXX_VERSION>=40700 && defined(__GXX_EXPERIMENTAL_CXX0X__) -/* Some C++11 features used inside libstdc++ are not supported by Intel compiler. - * Checking version of gcc instead of libstdc++ because - * - they are directly connected, - * - for now it is not possible to check version of any standard library in this file - */ +/* Some C++11 features used inside libstdc++ are not supported by Intel C++ Compiler. */ #define __TBB_ICC_13_0_CPP11_STDLIB_SUPPORT_BROKEN 1 #endif @@ -703,7 +738,7 @@ There are four cases that are supported: #endif #endif -/*In a PIC mode some versions of GCC 4.1.2 generate incorrect inlined code for 8 byte __sync_val_compare_and_swap intrinsic */ +/* In a PIC mode some versions of GCC 4.1.2 generate incorrect inlined code for 8 byte __sync_val_compare_and_swap intrinsic */ #if __TBB_GCC_VERSION == 40102 && __PIC__ && !defined(__INTEL_COMPILER) && !defined(__clang__) #define __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN 1 #endif @@ -734,16 +769,23 @@ There are four cases that are supported: #define __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN (_MSC_VER && (__INTEL_COMPILER >= 1300 && __INTEL_COMPILER <= 1310) && !__INTEL_CXX11_MODE__) #define __TBB_CPP11_DECLVAL_BROKEN (_MSC_VER == 1600 || (__GNUC__ && __TBB_GCC_VERSION < 40500) ) - -// Intel C++ compiler has difficulties with copying std::pair with VC11 std::reference_wrapper being a const member +// Intel C++ Compiler has difficulties with copying std::pair with VC11 std::reference_wrapper being a const member #define __TBB_COPY_FROM_NON_CONST_REF_BROKEN (_MSC_VER == 1700 && __INTEL_COMPILER && __INTEL_COMPILER < 1600) // The implicit upcasting of the tuple of a reference of a derived class to a base class fails on icc 13.X if the system's gcc environment is 4.8 // Also in gcc 4.4 standard library the implementation of the tuple<&> conversion (tuple a = tuple, B is inherited from A) is broken. -#if __GXX_EXPERIMENTAL_CXX0X__ && ((__INTEL_COMPILER >=1300 && __INTEL_COMPILER <=1310 && __TBB_GLIBCXX_VERSION>=40700) || (__TBB_GLIBCXX_VERSION < 40500)) +#if __GXX_EXPERIMENTAL_CXX0X__ && __GLIBCXX__ && ((__INTEL_COMPILER >=1300 && __INTEL_COMPILER <=1310 && __TBB_GLIBCXX_VERSION>=40700) || (__TBB_GLIBCXX_VERSION < 40500)) #define __TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN 1 #endif +// In some cases decltype of a function adds a reference to a return type. +#define __TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN (_MSC_VER == 1600 && !__INTEL_COMPILER) + +// Visual Studio 2013 does not delete the copy constructor when a user-defined move constructor is provided +#if _MSC_VER && _MSC_VER <= 1800 + #define __TBB_IMPLICIT_COPY_DELETION_BROKEN 1 +#endif + /** End of __TBB_XXX_BROKEN macro section **/ #if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) @@ -764,7 +806,7 @@ There are four cases that are supported: #define __TBB_VARIADIC_PARALLEL_INVOKE (TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) #define __TBB_FLOW_GRAPH_CPP11_FEATURES (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_AUTO_PRESENT) \ + && __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_AUTO_PRESENT) \ && __TBB_CPP11_VARIADIC_TUPLE_PRESENT && __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT \ && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN #define __TBB_PREVIEW_STREAMING_NODE (__TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT && __TBB_FLOW_GRAPH_CPP11_FEATURES \ @@ -773,7 +815,13 @@ There are four cases that are supported: #define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES || __TBB_PREVIEW_OPENCL_NODE) #define __TBB_PREVIEW_ASYNC_MSG (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_FLOW_GRAPH_CPP11_FEATURES) -#define __TBB_PREVIEW_GFX_FACTORY (__TBB_GFX_PRESENT && TBB_PREVIEW_FLOW_GRAPH_FEATURES && !__TBB_MIC_OFFLOAD \ - && __TBB_FLOW_GRAPH_CPP11_FEATURES && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT \ - && __TBB_CPP11_FUTURE_PRESENT) + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#define __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES TBB_PREVIEW_FLOW_GRAPH_FEATURES +#endif + +#ifndef __TBB_PREVIEW_CRITICAL_TASKS +#define __TBB_PREVIEW_CRITICAL_TASKS (__TBB_CPF_BUILD || __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES) +#endif + #endif /* __TBB_tbb_config_H */ diff --git a/inst/include/tbb_local/tbb/tbb_disable_exceptions.h b/inst/include/tbb_local/tbb/tbb_disable_exceptions.h index a1d28a60..88ef4351 100644 --- a/inst/include/tbb_local/tbb/tbb_disable_exceptions.h +++ b/inst/include/tbb_local/tbb/tbb_disable_exceptions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ //! To disable use of exceptions, include this header before any other header file from the library. diff --git a/inst/include/tbb_local/tbb/tbb_exception.h b/inst/include/tbb_local/tbb/tbb_exception.h index 1c843309..2dcb62b5 100644 --- a/inst/include/tbb_local/tbb/tbb_exception.h +++ b/inst/include/tbb_local/tbb/tbb_exception.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,31 +12,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_exception_H #define __TBB_exception_H #include "tbb_stddef.h" - -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include -#include //required for bad_alloc definition, operators new +#include // required for bad_alloc definition, operators new #include // required to construct std exception classes -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - namespace tbb { //! Exception for concurrent containers @@ -164,19 +149,19 @@ class tbb_exception : public std::exception //! Creates and returns pointer to the deep copy of this exception object. /** Move semantics is allowed. **/ - virtual tbb_exception* move () throw() = 0; + virtual tbb_exception* move() throw() = 0; //! Destroys objects created by the move() method. /** Frees memory and calls destructor for this exception object. Can and must be used only on objects created by the move method. **/ - virtual void destroy () throw() = 0; + virtual void destroy() throw() = 0; //! Throws this exception object. /** Make sure that if you have several levels of derivation from this interface you implement or override this method on the most derived level. The implementation is as simple as "throw *this;". Failure to do this will result in exception of a base class type being thrown. **/ - virtual void throw_self () = 0; + virtual void throw_self() = 0; //! Returns RTTI name of the originally intercepted exception virtual const char* name() const throw() = 0; @@ -203,19 +188,19 @@ class tbb_exception : public std::exception class captured_exception : public tbb_exception { public: - captured_exception ( const captured_exception& src ) + captured_exception( const captured_exception& src ) : tbb_exception(src), my_dynamic(false) { set(src.my_exception_name, src.my_exception_info); } - captured_exception ( const char* name_, const char* info ) + captured_exception( const char* name_, const char* info ) : my_dynamic(false) { set(name_, info); } - __TBB_EXPORTED_METHOD ~captured_exception () throw(); + __TBB_EXPORTED_METHOD ~captured_exception() throw(); captured_exception& operator= ( const captured_exception& src ) { if ( this != &src ) { @@ -225,25 +210,25 @@ class captured_exception : public tbb_exception return *this; } - captured_exception* __TBB_EXPORTED_METHOD move () throw() __TBB_override; + captured_exception* __TBB_EXPORTED_METHOD move() throw() __TBB_override; - void __TBB_EXPORTED_METHOD destroy () throw() __TBB_override; + void __TBB_EXPORTED_METHOD destroy() throw() __TBB_override; - void throw_self () __TBB_override { __TBB_THROW(*this); } + void throw_self() __TBB_override { __TBB_THROW(*this); } const char* __TBB_EXPORTED_METHOD name() const throw() __TBB_override; const char* __TBB_EXPORTED_METHOD what() const throw() __TBB_override; - void __TBB_EXPORTED_METHOD set ( const char* name, const char* info ) throw(); - void __TBB_EXPORTED_METHOD clear () throw(); + void __TBB_EXPORTED_METHOD set( const char* name, const char* info ) throw(); + void __TBB_EXPORTED_METHOD clear() throw(); private: - //! Used only by method clone(). + //! Used only by method move(). captured_exception() {} - //! Functionally equivalent to {captured_exception e(name,info); return e.clone();} - static captured_exception* allocate ( const char* name, const char* info ); + //! Functionally equivalent to {captured_exception e(name,info); return e.move();} + static captured_exception* allocate( const char* name, const char* info ); bool my_dynamic; const char* my_exception_name; @@ -261,7 +246,7 @@ class movable_exception : public tbb_exception typedef movable_exception self_type; public: - movable_exception ( const ExceptionData& data_ ) + movable_exception( const ExceptionData& data_ ) : my_exception_data(data_) , my_dynamic(false) , my_exception_name( @@ -273,14 +258,14 @@ class movable_exception : public tbb_exception ) {} - movable_exception ( const movable_exception& src ) throw () + movable_exception( const movable_exception& src ) throw () : tbb_exception(src) , my_exception_data(src.my_exception_data) , my_dynamic(false) , my_exception_name(src.my_exception_name) {} - ~movable_exception () throw() {} + ~movable_exception() throw() {} const movable_exception& operator= ( const movable_exception& src ) { if ( this != &src ) { @@ -290,15 +275,15 @@ class movable_exception : public tbb_exception return *this; } - ExceptionData& data () throw() { return my_exception_data; } + ExceptionData& data() throw() { return my_exception_data; } - const ExceptionData& data () const throw() { return my_exception_data; } + const ExceptionData& data() const throw() { return my_exception_data; } - const char* name () const throw() __TBB_override { return my_exception_name; } + const char* name() const throw() __TBB_override { return my_exception_name; } - const char* what () const throw() __TBB_override { return "tbb::movable_exception"; } + const char* what() const throw() __TBB_override { return "tbb::movable_exception"; } - movable_exception* move () throw() __TBB_override { + movable_exception* move() throw() __TBB_override { void* e = internal::allocate_via_handler_v3(sizeof(movable_exception)); if ( e ) { ::new (e) movable_exception(*this); @@ -306,14 +291,14 @@ class movable_exception : public tbb_exception } return (movable_exception*)e; } - void destroy () throw() __TBB_override { + void destroy() throw() __TBB_override { __TBB_ASSERT ( my_dynamic, "Method destroy can be called only on dynamically allocated movable_exceptions" ); if ( my_dynamic ) { this->~movable_exception(); internal::deallocate_via_handler_v3(this); } } - void throw_self () __TBB_override { __TBB_THROW( *this ); } + void throw_self() __TBB_override { __TBB_THROW( *this ); } protected: //! User data @@ -333,26 +318,26 @@ namespace internal { //! Exception container that preserves the exact copy of the original exception /** This class can be used only when the appropriate runtime support (mandated - by C++0x) is present **/ + by C++11) is present **/ class tbb_exception_ptr { std::exception_ptr my_ptr; public: - static tbb_exception_ptr* allocate (); - static tbb_exception_ptr* allocate ( const tbb_exception& tag ); + static tbb_exception_ptr* allocate(); + static tbb_exception_ptr* allocate( const tbb_exception& tag ); //! This overload uses move semantics (i.e. it empties src) - static tbb_exception_ptr* allocate ( captured_exception& src ); + static tbb_exception_ptr* allocate( captured_exception& src ); //! Destroys this objects /** Note that objects of this type can be created only by the allocate() method. **/ - void destroy () throw(); + void destroy() throw(); //! Throws the contained exception . - void throw_self () { std::rethrow_exception(my_ptr); } + void throw_self() { std::rethrow_exception(my_ptr); } private: - tbb_exception_ptr ( const std::exception_ptr& src ) : my_ptr(src) {} - tbb_exception_ptr ( const captured_exception& src ) : + tbb_exception_ptr( const std::exception_ptr& src ) : my_ptr(src) {} + tbb_exception_ptr( const captured_exception& src ) : #if __TBB_MAKE_EXCEPTION_PTR_PRESENT my_ptr(std::make_exception_ptr(src)) // the final function name in C++11 #else diff --git a/inst/include/tbb_local/tbb/tbb_machine.h b/inst/include/tbb_local/tbb/tbb_machine.h index 68d1d5d0..095b9d3c 100644 --- a/inst/include/tbb_local/tbb/tbb_machine.h +++ b/inst/include/tbb_local/tbb/tbb_machine.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_H @@ -227,7 +223,7 @@ template<> struct atomic_selector<8> { #include "machine/linux_intel64.h" #endif -#elif __linux__ || __FreeBSD__ || __NetBSD__ +#elif __linux__ || __FreeBSD__ || __NetBSD__ || __OpenBSD__ #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) #include "machine/gcc_generic.h" @@ -241,8 +237,8 @@ template<> struct atomic_selector<8> { #include "machine/linux_ia64.h" #elif __powerpc__ #include "machine/mac_ppc.h" - #elif __ARM_ARCH_7A__ - #include "machine/gcc_armv7.h" + #elif __ARM_ARCH_7A__ || __aarch64__ + #include "machine/gcc_arm.h" #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT #include "machine/gcc_generic.h" #endif @@ -663,7 +659,15 @@ struct machine_load_store_seq_cst { return __TBB_machine_cmpswp8( (volatile void*)const_cast(&location), anyvalue, anyvalue ); } static void store ( volatile T &location, T value ) { +#if __TBB_GCC_VERSION >= 40702 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + // An atomic initialization leads to reading of uninitialized memory int64_t result = (volatile int64_t&)location; +#if __TBB_GCC_VERSION >= 40702 +#pragma GCC diagnostic pop +#endif while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result ) result = (volatile int64_t&)location; } @@ -780,7 +784,7 @@ struct __TBB_machine_type_with_alignment_##PowerOf2 { \ #endif /* Now declare types aligned to useful powers of two */ -// TODO: Is __TBB_DefineTypeWithAlignment(8) needed on 32 bit platforms? +__TBB_DefineTypeWithAlignment(8) // i386 ABI says that uint64_t is aligned on 4 bytes __TBB_DefineTypeWithAlignment(16) __TBB_DefineTypeWithAlignment(32) __TBB_DefineTypeWithAlignment(64) @@ -794,7 +798,7 @@ template struct type_with_alignment; template<> struct type_with_alignment<1> { char member; }; template<> struct type_with_alignment<2> { uint16_t member; }; template<> struct type_with_alignment<4> { uint32_t member; }; -template<> struct type_with_alignment<8> { uint64_t member; }; +template<> struct type_with_alignment<8> { __TBB_machine_type_with_alignment_8 member; }; template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; }; template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; }; template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; }; diff --git a/inst/include/tbb_local/tbb/tbb_profiling.h b/inst/include/tbb_local/tbb/tbb_profiling.h index ffaf98b9..21665800 100644 --- a/inst/include/tbb_local/tbb/tbb_profiling.h +++ b/inst/include/tbb_local/tbb/tbb_profiling.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_profiling_H @@ -24,10 +20,6 @@ namespace tbb { namespace internal { - // - // This is not under __TBB_ITT_STRUCTURE_API because these values are used directly in flow_graph.h. - // - // include list of index names #define TBB_STRING_RESOURCE(index_name,str) index_name, enum string_index { @@ -138,8 +130,7 @@ namespace tbb { void __TBB_EXPORTED_FUNC itt_store_pointer_with_release_v3(void *dst, void *src); void* __TBB_EXPORTED_FUNC itt_load_pointer_with_acquire_v3(const void *src); void* __TBB_EXPORTED_FUNC itt_load_pointer_v3( const void* src ); -#if __TBB_ITT_STRUCTURE_API - enum itt_domain_enum { ITT_DOMAIN_FLOW=0 }; + enum itt_domain_enum { ITT_DOMAIN_FLOW=0, ITT_DOMAIN_MAIN=1, ITT_DOMAIN_ALGO=2, ITT_NUM_DOMAINS }; void __TBB_EXPORTED_FUNC itt_make_task_group_v7( itt_domain_enum domain, void *group, unsigned long long group_extra, void *parent, unsigned long long parent_extra, string_index name_index ); @@ -154,7 +145,6 @@ namespace tbb { void __TBB_EXPORTED_FUNC itt_region_begin_v9( itt_domain_enum domain, void *region, unsigned long long region_extra, void *parent, unsigned long long parent_extra, string_index name_index ); void __TBB_EXPORTED_FUNC itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long region_extra ); -#endif // __TBB_ITT_STRUCTURE_API // two template arguments are to workaround /Wp64 warning with tbb::atomic specialized for unsigned type template @@ -238,12 +228,6 @@ namespace tbb { call_itt_notify_v5((int)t, ptr); } -#else - inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} - -#endif // TBB_USE_THREADING_TOOLS - -#if __TBB_ITT_STRUCTURE_API inline void itt_make_task_group( itt_domain_enum domain, void *group, unsigned long long group_extra, void *parent, unsigned long long parent_extra, string_index name_index ) { itt_make_task_group_v7( domain, group, group_extra, parent, parent_extra, name_index ); @@ -276,9 +260,81 @@ namespace tbb { inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { itt_region_end_v9( domain, region, region_extra ); } -#endif // __TBB_ITT_STRUCTURE_API +#else + inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {} + + inline void itt_make_task_group( itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_metadata_str_add( itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, + string_index /*key*/, const char* /*value*/ ) {} + + inline void itt_relation_add( itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/, + itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) {} + + inline void itt_task_begin( itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_task_end( itt_domain_enum /*domain*/ ) {} + + inline void itt_region_begin( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_region_end( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) {} +#endif // TBB_USE_THREADING_TOOLS } // namespace internal } // namespace tbb +#if TBB_PREVIEW_FLOW_GRAPH_TRACE +#include + +namespace tbb { +namespace profiling { +namespace interface10 { + +#if TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) +class event { +/** This class supports user event traces through itt. + Common use-case is tagging data flow graph tasks (data-id) + and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/ +// TODO: Replace implementation by itt user event api. + + const std::string my_name; + + static void emit_trace(const std::string &input) { + itt_metadata_str_add( tbb::internal::ITT_DOMAIN_FLOW, NULL, tbb::internal::FLOW_NULL, tbb::internal::USER_EVENT, ( "FGA::DATAID::" + input ).c_str() ); + } + +public: + event(const std::string &input) + : my_name( input ) + { } + + void emit() { + emit_trace(my_name); + } + + static void emit(const std::string &description) { + emit_trace(description); + } + +}; +#else // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) +// Using empty struct if user event tracing is disabled: +struct event { + event(const std::string &) { } + + void emit() { } + + static void emit(const std::string &) { } +}; +#endif // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) + +} // interfaceX +using interface10::event; +} // namespace profiling +} // namespace tbb +#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE + #endif /* __TBB_profiling_H */ diff --git a/inst/include/tbb_local/tbb/tbb_stddef.h b/inst/include/tbb_local/tbb/tbb_stddef.h index 236f3d83..bb3233db 100644 --- a/inst/include/tbb_local/tbb/tbb_stddef.h +++ b/inst/include/tbb_local/tbb/tbb_stddef.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,21 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_stddef_H #define __TBB_tbb_stddef_H // Marketing-driven product version -#define TBB_VERSION_MAJOR 2017 +#define TBB_VERSION_MAJOR 2019 #define TBB_VERSION_MINOR 0 // Engineering-focused interface version -#define TBB_INTERFACE_VERSION 9107 +#define TBB_INTERFACE_VERSION 11008 #define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 // The oldest major interface version still supported @@ -155,8 +151,8 @@ namespace tbb { #if TBB_USE_ASSERT - //! Assert that x is true. - /** If x is false, print assertion failure message. + //! Assert that predicate is true. + /** If predicate is false, print assertion failure message. If the comment argument is not NULL, it is printed as part of the failure message. The comment argument has no other effect. */ #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) @@ -246,6 +242,14 @@ const size_t NFS_MaxLineSize = 128; #define __TBB_override // formal comment only #endif +#if __TBB_CPP17_FALLTHROUGH_PRESENT +#define __TBB_fallthrough [[fallthrough]] +#elif __TBB_FALLTHROUGH_PRESENT +#define __TBB_fallthrough __attribute__ ((fallthrough)) +#else +#define __TBB_fallthrough +#endif + template struct padded_base : T { char pad[S - R]; @@ -278,7 +282,7 @@ void __TBB_EXPORTED_FUNC handle_perror( int error_code, const char* aux_info ); inline bool __TBB_false() { return false; } #define __TBB_TRY #define __TBB_CATCH(e) if ( tbb::internal::__TBB_false() ) - #define __TBB_THROW(e) ((void)0) + #define __TBB_THROW(e) tbb::internal::suppress_unused_warning(e) #define __TBB_RETHROW() ((void)0) #endif /* !TBB_USE_EXCEPTIONS */ @@ -326,7 +330,7 @@ class no_assign { //! Base class for types that should not be copied or assigned. class no_copy: no_assign { //! Deny copy construction - no_copy( const no_copy& ); + //no_copy( const no_copy& ); public: //! Allow default construction no_copy() {} @@ -423,17 +427,24 @@ class proportional_split: internal::no_assign { // Following is a set of classes and functions typically used in compile-time "metaprogramming". // TODO: move all that to a separate header -#if __TBB_ALLOCATOR_TRAITS_PRESENT -#include //for allocator_traits +#if __TBB_CPP11_SMART_POINTERS_PRESENT +#include // for unique_ptr #endif -#if __TBB_CPP11_RVALUE_REF_PRESENT || _LIBCPP_VERSION -#include // for std::move +#if __TBB_CPP11_RVALUE_REF_PRESENT || __TBB_CPP11_DECLTYPE_PRESENT || _LIBCPP_VERSION +#include // for std::move, std::forward, std::declval #endif namespace tbb { namespace internal { +#if __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + template + std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); + } +#endif + //! Class for determining type of std::allocator::value_type. template struct allocator_type { @@ -457,15 +468,6 @@ struct bool_constant { typedef bool_constant true_type; typedef bool_constant false_type; -#if __TBB_ALLOCATOR_TRAITS_PRESENT -using std::allocator_traits; -#else -template -struct allocator_traits{ - typedef tbb::internal::false_type propagate_on_container_move_assignment; -}; -#endif - //! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. template struct select_size_t_constant { @@ -526,7 +528,7 @@ struct STATIC_ASSERTION_FAILED; //intentionally left undefined to cause co //! @endcond }} // namespace tbb::internal -#if __TBB_STATIC_ASSERT_PRESENT +#if __TBB_STATIC_ASSERT_PRESENT #define __TBB_STATIC_ASSERT(condition,msg) static_assert(condition,msg) #else //please note condition is intentionally inverted to get a bit more understandable error msg diff --git a/inst/include/tbb_local/tbb/tbb_thread.h b/inst/include/tbb_local/tbb/tbb_thread.h index d1cafd65..52b96e4d 100644 --- a/inst/include/tbb_local/tbb/tbb_thread.h +++ b/inst/include/tbb_local/tbb/tbb_thread.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_thread_H @@ -47,19 +43,9 @@ namespace tbb { namespace internal { #include "internal/_tbb_hash_compare_impl.h" #include "tick_count.h" -#if !TBB_USE_EXCEPTIONS && _MSC_VER - // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers - #pragma warning (push) - #pragma warning (disable: 4530) -#endif - #include __TBB_STD_SWAP_HEADER #include -#if !TBB_USE_EXCEPTIONS && _MSC_VER - #pragma warning (pop) -#endif - namespace tbb { namespace internal { @@ -256,7 +242,7 @@ namespace internal { friend tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); friend inline size_t tbb_hasher( const tbb_thread_v3::id& id ) { - __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementaion assumes that thread_id_type fits into machine word"); + __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementation assumes that thread_id_type fits into machine word"); return tbb::tbb_hasher(id.my_id); } diff --git a/inst/include/tbb_local/tbb/tbbmalloc_proxy.h b/inst/include/tbb_local/tbb/tbbmalloc_proxy.h index 76cbd6d7..d7b7235a 100644 --- a/inst/include/tbb_local/tbb/tbbmalloc_proxy.h +++ b/inst/include/tbb_local/tbb/tbbmalloc_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* @@ -63,4 +59,7 @@ struct __TBB_malloc_proxy_caller { #endif // _MSC_VER +/* Public Windows API */ +extern "C" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr); + #endif //__TBB_tbbmalloc_proxy_H diff --git a/inst/include/tbb_local/tbb/tick_count.h b/inst/include/tbb_local/tbb/tick_count.h index a7f4e0f3..a3c744ce 100644 --- a/inst/include/tbb_local/tbb/tick_count.h +++ b/inst/include/tbb_local/tbb/tick_count.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tick_count_H diff --git a/src/tbb/CHANGES b/src/tbb/CHANGES index aa7f126f..6e7cb60f 100644 --- a/src/tbb/CHANGES +++ b/src/tbb/CHANGES @@ -2,6 +2,367 @@ The list of most significant changes made over time in Intel(R) Threading Building Blocks (Intel(R) TBB). +Intel TBB 2019 Update 8 +TBB_INTERFACE_VERSION == 11008 + +Changes (w.r.t. Intel TBB 2019 Update 7): + +Bugs fixed: + +- Fixed a bug in TBB 2019 Update 7 that could lead to incorrect memory + reallocation on Linux (https://github.com/intel/tbb/issues/148). +- Fixed enqueuing tbb::task into tbb::task_arena not to fail on threads + with no task scheduler initialized + (https://github.com/intel/tbb/issues/116). + +------------------------------------------------------------------------ +Intel TBB 2019 Update 7 +TBB_INTERFACE_VERSION == 11007 + +Changes (w.r.t. Intel TBB 2019 Update 6): + +- Added TBBMALLOC_SET_HUGE_SIZE_THRESHOLD parameter to set the lower + bound for allocations that are not released back to OS unless + a cleanup is explicitly requested. +- Added zip_iterator::base() method to get the tuple of underlying + iterators. +- Improved async_node to never block a thread that sends a message + through its gateway. +- Extended decrement port of the tbb::flow::limiter_node to accept + messages of integral types. +- Added support of Windows* to the CMake module TBBInstallConfig. +- Added packaging of CMake configuration files to TBB packages built + using build/build.py script + (https://github.com/intel/tbb/issues/141). + +Changes affecting backward compatibility: + +- Removed the number_of_decrement_predecessors parameter from the + constructor of flow::limiter_node. To allow its usage, set + TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR macro to 1. + +Preview Features: + +- Added ordered associative containers: + concurrent_{map,multimap,set,multiset} (requires C++11). + +Open-source contributions integrated: + +- Fixed makefiles to properly obtain the GCC version for GCC 7 + and later (https://github.com/intel/tbb/pull/147) by Timmmm. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 6 +TBB_INTERFACE_VERSION == 11006 + +Changes (w.r.t. Intel TBB 2019 Update 5): + +- Added support for Microsoft* Visual Studio* 2019. +- Added support for enqueuing tbb::task into tbb::task_arena + (https://github.com/01org/tbb/issues/116). +- Improved support for allocator propagation on concurrent_hash_map + assigning and swapping. +- Improved scalable_allocation_command cleanup operations to release + more memory buffered by the calling thread. +- Separated allocation of small and large objects into distinct memory + regions, which helps to reduce excessive memory caching inside the + TBB allocator. + +Preview Features: + +- Removed template class gfx_factory from the flow graph API. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 5 +TBB_INTERFACE_VERSION == 11005 + +Changes (w.r.t. Intel TBB 2019 Update 4): + +- Associating a task_scheduler_observer with an implicit or explicit + task arena is now a fully supported feature. +- Added a CMake module TBBInstallConfig that allows to generate and + install CMake configuration files for TBB packages. + Inspired by Hans Johnson (https://github.com/01org/tbb/pull/119). +- Added node handles, methods merge() and unsafe_extract() to concurrent + unordered containers. +- Added constructors with Compare argument to concurrent_priority_queue + (https://github.com/01org/tbb/issues/109). +- Controlling the stack size of worker threads is now supported for + Universal Windows Platform. +- Improved tbb::zip_iterator to work with algorithms that swap values + via iterators. +- Improved support for user-specified allocators in concurrent_hash_map, + including construction of allocator-aware data types. +- For ReaderWriterMutex types, upgrades and downgrades now succeed if + the mutex is already in the requested state. + Inspired by Niadb (https://github.com/01org/tbb/pull/122). + +Preview Features: + +- The task_scheduler_observer::may_sleep() method has been removed. + +Bugs fixed: + +- Fixed the issue with a pipeline parallel filter executing serially if + it follows a thread-bound filter. +- Fixed a performance regression observed when multiple parallel + algorithms start simultaneously. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 4 +TBB_INTERFACE_VERSION == 11004 + +Changes (w.r.t. Intel TBB 2019 Update 3): + +- global_control class is now a fully supported feature. +- Added deduction guides for tbb containers: concurrent_hash_map, + concurrent_unordered_map, concurrent_unordered_set. +- Added tbb::scalable_memory_resource function returning + std::pmr::memory_resource interface to the TBB memory allocator. +- Added tbb::cache_aligned_resource class that implements + std::pmr::memory_resource with cache alignment and no false sharing. +- Added rml::pool_msize function returning the usable size of a memory + block allocated from a given memory pool. +- Added default and copy constructors for tbb::counting_iterator + and tbb::zip_iterator. +- Added TBB_malloc_replacement_log function to obtain the status of + dynamic memory allocation replacement (Windows* only). +- CMake configuration file now supports release-only and debug-only + configurations (https://github.com/01org/tbb/issues/113). +- TBBBuild CMake module takes the C++ version from CMAKE_CXX_STANDARD. + +Bugs fixed: + +- Fixed compilation for tbb::concurrent_vector when used with + std::pmr::polymorphic_allocator. + +Open-source contributions integrated: + +- TBB_INTERFACE_VERSION is included into TBB version in CMake + configuration (https://github.com/01org/tbb/pull/100) + by Hans Johnson. +- Fixed detection of C++17 deduction guides for Visual C++* + (https://github.com/01org/tbb/pull/112) by Marian Klymov. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 3 +TBB_INTERFACE_VERSION == 11003 + +Changes (w.r.t. Intel TBB 2019 Update 2): + +- Added tbb::transform_iterator. +- Added new Makefile target 'profile' to flow graph examples enabling + additional support for Intel(R) Parallel Studio XE tools. +- Added TBB_MALLOC_DISABLE_REPLACEMENT environment variable to switch off + dynamic memory allocation replacement on Windows*. Inspired by + a contribution from Edward Lam. + +Preview Features: + +- Extended flow graph API to support relative priorities for functional + nodes, specified as an optional parameter to the node constructors. + +Open-source contributions integrated: + +- Enabled using process-local futex operations + (https://github.com/01org/tbb/pull/58) by Andrey Semashev. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 2 +TBB_INTERFACE_VERSION == 11002 + +Changes (w.r.t. Intel TBB 2019 Update 1): + +- Added overloads for parallel_reduce with default partitioner and + user-supplied context. +- Added deduction guides for tbb containers: concurrent_vector, + concurrent_queue, concurrent_bounded_queue, + concurrent_priority_queue. +- Reallocation of memory objects >1MB now copies and frees memory if + the size is decreased twice or more, trading performance off for + reduced memory usage. +- After a period of sleep, TBB worker threads now prefer returning to + their last used task arena. + +Bugs fixed: + +- Fixed compilation of task_group.h when targeting macOS* 10.11 or + earlier (https://github.com/conda-forge/tbb-feedstock/issues/42). + +Open-source contributions integrated: + +- Added constructors with HashCompare argument to concurrent_hash_map + (https://github.com/01org/tbb/pull/63) by arewedancer. + +------------------------------------------------------------------------ +Intel TBB 2019 Update 1 +TBB_INTERFACE_VERSION == 11001 + +Changes (w.r.t. Intel TBB 2019): + +- Doxygen documentation could be built with 'make doxygen' command now. + +Changes affecting backward compatibility: + +- Enforced 8 byte alignment for tbb::atomic and + tbb::atomic. On IA-32 architecture it may cause layout + changes in structures that use these types. + +Bugs fixed: + +- Fixed an issue with dynamic memory allocation replacement on Windows* + occurred for some versions of ucrtbase.dll. +- Fixed possible deadlock in tbbmalloc cleanup procedure during process + shutdown. Inspired by a contribution from Edward Lam. +- Fixed usage of std::uncaught_exception() deprecated in C++17 + (https://github.com/01org/tbb/issues/67). +- Fixed a crash when a local observer is activated after an arena + observer. +- Fixed compilation of task_group.h by Visual C++* 15.7 with + /permissive- option (https://github.com/01org/tbb/issues/53). +- Fixed tbb4py to avoid dependency on Intel(R) C++ Compiler shared + libraries. +- Fixed compilation for Anaconda environment with GCC 7.3 and higher. + +Open-source contributions integrated: + +- Fix various warnings when building with Visual C++ + (https://github.com/01org/tbb/pull/70) by Edward Lam. + +------------------------------------------------------------------------ +Intel TBB 2019 +TBB_INTERFACE_VERSION == 11000 + +Changes (w.r.t. Intel TBB 2018 Update 5): + +- Lightweight policy for functional nodes in the flow graph is now + a fully supported feature. +- Reservation support in flow::write_once_node and flow::overwrite_node + is now a fully supported feature. +- Support for Flow Graph Analyzer and improvements for + Intel(R) VTune(TM) Amplifier become a regular feature enabled by + TBB_USE_THREADING_TOOLS macro. +- Added support for std::new_handler in the replacement functions for + global operator new. +- Added C++14 constructors to concurrent unordered containers. +- Added tbb::counting_iterator and tbb::zip_iterator. +- Fixed multiple -Wextra warnings in TBB source files. + +Preview Features: + +- Extracting nodes from a flow graph is deprecated and disabled by + default. To enable, use TBB_DEPRECATED_FLOW_NODE_EXTRACTION macro. + +Changes affecting backward compatibility: + +- Due to internal changes in the flow graph classes, recompilation is + recommended for all binaries that use the flow graph. + +Open-source contributions integrated: + +- Added support for OpenBSD by Anthony J. Bentley. + +------------------------------------------------------------------------ +Intel TBB 2018 Update 6 +TBB_INTERFACE_VERSION == 10006 + +Changes (w.r.t. Intel TBB 2018 Update 5): + +Bugs fixed: + +- Fixed an issue with dynamic memory allocation replacement on Windows* + occurred for some versions of ucrtbase.dll. + +------------------------------------------------------------------------ +Intel TBB 2018 Update 5 +TBB_INTERFACE_VERSION == 10005 + +Changes (w.r.t. Intel TBB 2018 Update 4): + +Preview Features: + +- Added user event tracing API for Intel(R) VTune(TM) Amplifier and + Flow Graph Analyzer. + +Bugs fixed: + +- Fixed the memory allocator to properly support transparent huge pages. +- Removed dynamic exception specifications in tbbmalloc_proxy for C++11 + and later (https://github.com/01org/tbb/issues/41). +- Added -flifetime-dse=1 option when building with GCC on macOS* + (https://github.com/01org/tbb/issues/60). + +Open-source contributions integrated: + +- Added ARMv8 support by Siddhesh Poyarekar. +- Avoid GCC warnings for clearing an object of non-trivial type + (https://github.com/01org/tbb/issues/54) by Daniel Arndt. + +------------------------------------------------------------------------ +Intel TBB 2018 Update 4 +TBB_INTERFACE_VERSION == 10004 + +Changes (w.r.t. Intel TBB 2018 Update 3): + +Preview Features: + +- Improved support for Flow Graph Analyzer and Intel(R) VTune(TM) + Amplifier in the task scheduler and generic parallel algorithms. +- Default device set for opencl_node now includes all the devices from + the first available OpenCL* platform. +- Added lightweight policy for functional nodes in the flow graph. It + indicates that the node body has little work and should, if possible, + be executed immediately upon receiving a message, avoiding task + scheduling overhead. + +------------------------------------------------------------------------ +Intel TBB 2018 Update 3 +TBB_INTERFACE_VERSION == 10003 + +Changes (w.r.t. Intel TBB 2018 Update 2): + +Preview Features: + +- Added template class blocked_rangeNd for a generic multi-dimensional + range (requires C++11). Inspired by a contribution from Jeff Hammond. + +Bugs fixed: + +- Fixed a crash with dynamic memory allocation replacement on + Windows* for applications using system() function. +- Fixed parallel_deterministic_reduce to split range correctly when used + with static_partitioner. +- Fixed a synchronization issue in task_group::run_and_wait() which + caused a simultaneous call to task_group::wait() to return + prematurely. + +------------------------------------------------------------------------ +Intel TBB 2018 Update 2 +TBB_INTERFACE_VERSION == 10002 + +Changes (w.r.t. Intel TBB 2018 Update 1): + +- Added support for Android* NDK r16, macOS* 10.13, Fedora* 26. +- Binaries for Universal Windows Driver (vc14_uwd) now link with static + Microsoft* runtime libraries, and are only available in commercial + releases. +- Extended flow graph documentation with more code samples. + +Preview Features: + +- Added a Python* module for multi-processing computations in numeric + Python* libraries. + +Bugs fixed: + +- Fixed constructors of concurrent_hash_map to be exception-safe. +- Fixed auto-initialization in the main thread to be cleaned up at + shutdown. +- Fixed a crash when tbbmalloc_proxy is used together with dbghelp. +- Fixed static_partitioner to assign tasks properly in case of nested + parallelism. + +------------------------------------------------------------------------ Intel TBB 2018 Update 1 TBB_INTERFACE_VERSION == 10001 @@ -235,7 +596,7 @@ Changes (w.r.t. Intel TBB 4.4 Update 5): - Added TBB_USE_GLIBCXX_VERSION macro to specify the version of GNU libstdc++ when it cannot be properly recognized, e.g. when used with Clang on Linux* OS. Inspired by a contribution from David A. -- Added graph/stereo example to demostrate tbb::flow::async_msg. +- Added graph/stereo example to demonstrate tbb::flow::async_msg. - Removed a few cases of excessive user data copying in the flow graph. - Reworked split_node to eliminate unnecessary overheads. - Added support for C++11 move semantics to the argument of @@ -946,7 +1307,7 @@ Changes (w.r.t. Intel TBB 4.1 Update 4): were added on OS X*. - For OS X* exact exception propagation is supported with Clang; it requires use of libc++ and corresponding Intel TBB binaries. -- Support for C++11 initializer lists in constructor and assigment +- Support for C++11 initializer lists in constructor and assignment has been added to concurrent_hash_map, concurrent_unordered_set, concurrent_unordered_multiset, concurrent_unordered_map, concurrent_unordered_multimap. @@ -1542,7 +1903,7 @@ Changes (w.r.t. 20100310 open-source release): - Reworked enumerable_thread_specific to use a custom implementation of hash map that is more efficient for ETS usage models. - Added example for class task_group; see examples/task_group/sudoku. -- Removed two examples, as they were long outdated and superceded: +- Removed two examples, as they were long outdated and superseded: pipeline/text_filter (use pipeline/square); parallel_while/parallel_preorder (use parallel_do/parallel_preorder). - PDF documentation updated. @@ -1760,7 +2121,7 @@ Changes (w.r.t. Intel TBB 2.1 U3 commercial-aligned release): - Added tests for aligned memory allocations and malloc replacement. - Several improvements for better bundling with Intel(R) C++ Compiler. -- A few other small changes in code and documentaion. +- A few other small changes in code and documentation. Bugs fixed: diff --git a/src/tbb/Doxyfile b/src/tbb/Doxyfile new file mode 100644 index 00000000..3c7727f8 --- /dev/null +++ b/src/tbb/Doxyfile @@ -0,0 +1,1325 @@ +# Doxyfile 1.4.7 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = "Intel(R) Threading Building Blocks Doxygen Documentation" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = "version 4.2.3" + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, +# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, +# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, +# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, +# Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# This tag can be used to specify the encoding used in the generated output. +# The encoding is not always determined by the language that is chosen, +# but also whether or not the output is meant for Windows or non-Windows users. +# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES +# forces the Windows encoding (this is the default for the Windows binary), +# whereas setting the tag to NO uses a Unix-style encoding (the default for +# all platforms other than Windows). + +USE_WINDOWS_ENCODING = NO + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = YES + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like the Qt-style comments (thus requiring an +# explicit @brief command for a brief description. + +JAVADOC_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 0 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = YES + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = NO + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = NO + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = NO + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = INTERNAL + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = include/ src/tbb/ + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = YES + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = NO + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = doc/copyright_brand_disclaimer_doxygen.txt + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = YES + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = NO + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = TBB_PREVIEW_FLOW_GRAPH_FEATURES \ + TBB_PREVIEW_FLOW_GRAPH_NODES \ + __TBB_PREVIEW_OPENCL_NODE \ + __TBB_CPP11_RVALUE_REF_PRESENT \ + __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ + __TBB_IMPLICIT_MOVE_PRESENT \ + __TBB_EXCEPTION_PTR_PRESENT \ + __TBB_STATIC_ASSERT_PRESENT \ + __TBB_CPP11_TUPLE_PRESENT \ + __TBB_INITIALIZER_LISTS_PRESENT \ + __TBB_CONSTEXPR_PRESENT \ + __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT \ + __TBB_NOEXCEPT_PRESENT \ + __TBB_CPP11_STD_BEGIN_END_PRESENT \ + __TBB_CPP11_AUTO_PRESENT \ + __TBB_CPP11_DECLTYPE_PRESENT \ + __TBB_CPP11_LAMBDAS_PRESENT \ + __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT \ + __TBB_OVERRIDE_PRESENT \ + __TBB_ALIGNAS_PRESENT \ + __TBB_CPP11_TEMPLATE_ALIASES_PRESENT \ + __TBB_FLOW_GRAPH_CPP11_FEATURES \ + __TBB_PREVIEW_STREAMING_NODE + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = YES + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = YES + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = YES + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = YES + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = svg + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# +# Note that this requires a modern browser other than Internet Explorer. Tested +# and working are Firefox, Chrome, Safari, and Opera. +# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make +# the SVG files visible. Older versions of IE do not have SVG support. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +INTERACTIVE_SVG = YES + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_WIDTH = 1024 + +# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_HEIGHT = 1024 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that a graph may be further truncated if the graph's +# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH +# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), +# the graph is not depth-constrained. + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes +# that will be shown in the graph. If the number of nodes in a graph becomes +# larger than this value, doxygen will truncate the graph, which is visualized +# by representing a node as a red box. Note that doxygen if the number of direct +# children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that +# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. +# Minimum value: 0, maximum value: 10000, default value: 50. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_GRAPH_MAX_NODES = 200 + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = YES diff --git a/src/tbb/LICENSE b/src/tbb/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/src/tbb/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/tbb/Makefile b/src/tbb/Makefile index d7dd56f9..29e3f5cf 100644 --- a/src/tbb/Makefile +++ b/src/tbb/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# tbb_root?=. include $(tbb_root)/build/common.inc diff --git a/src/tbb/README.md b/src/tbb/README.md new file mode 100644 index 00000000..4efa64a6 --- /dev/null +++ b/src/tbb/README.md @@ -0,0 +1,34 @@ +# Threading Building Blocks 2019 Update 8 +[![Stable release](https://img.shields.io/badge/version-2019_U8-green.svg)](https://github.com/01org/tbb/releases/tag/2019_U8) +[![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE) + +Threading Building Blocks (TBB) lets you easily write parallel C++ programs that take +full advantage of multicore performance, that are portable, composable and have future-proof scalability. + +## Release Information +Here are the latest [Changes](CHANGES) and [Release Notes](doc/Release_Notes.txt) (contains system requirements and known issues). + +Since [2018 U5](https://github.com/01org/tbb/releases/tag/2018_U5) TBB binary packages include [Parallel STL](https://github.com/intel/parallelstl) as a high-level component. + +## Documentation +* TBB [tutorial](https://software.intel.com/en-us/tbb-tutorial) +* TBB general documentation: [stable](https://software.intel.com/en-us/tbb-documentation) +and [latest](https://www.threadingbuildingblocks.org/docs/help/index.htm) + +## Support +Please report issues and suggestions via +[GitHub issues](https://github.com/01org/tbb/issues) or start a topic on the +[TBB forum](http://software.intel.com/en-us/forums/intel-threading-building-blocks/). + +## How to Contribute +To contribute to TBB, please open a GitHub pull request (preferred) or send us a patch by e-mail. +Threading Building Blocks is licensed under [Apache License, Version 2.0](LICENSE). +By its terms, contributions submitted to the project are also done under that license. + +## Engineering team contacts +* [E-mail us.](mailto:inteltbbdevelopers@intel.com) + +------------------------------------------------------------------------ +Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries. + +\* Other names and brands may be claimed as the property of others. diff --git a/src/tbb/build/AIX.gcc.inc b/src/tbb/build/AIX.gcc.inc index 18e6e6ad..4258a708 100644 --- a/src/tbb/build/AIX.gcc.inc +++ b/src/tbb/build/AIX.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ diff --git a/src/tbb/build/AIX.inc b/src/tbb/build/AIX.inc index abe12d52..45e3651c 100644 --- a/src/tbb/build/AIX.inc +++ b/src/tbb/build/AIX.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifndef arch arch:=$(shell uname -p) @@ -22,7 +18,7 @@ ifndef arch endif ifndef runtime - gcc_version:=$(shell gcc -dumpversion) + gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) diff --git a/src/tbb/build/BSD.clang.inc b/src/tbb/build/BSD.clang.inc new file mode 100644 index 00000000..042c09b0 --- /dev/null +++ b/src/tbb/build/BSD.clang.inc @@ -0,0 +1,106 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPILE_ONLY = -c -MMD +PREPROC_ONLY = -E -x c++ +INCLUDE_KEY = -I +DEFINE_KEY = -D +OUTPUT_KEY = -o # +OUTPUTOBJ_KEY = -o # +PIC_KEY = -fPIC +WARNING_AS_ERROR_KEY = -Werror +WARNING_KEY = -Wall +TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor +WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else +DYLIB_KEY = -shared +EXPORT_KEY = -Wl,--version-script, +LIBDL = + +CPLUS = clang++ +CONLY = clang +LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) +LIBS += -lpthread +LINK_FLAGS = -Wl,-rpath-link=. -Wl,-rpath=. -rdynamic +C_FLAGS = $(CPLUS_FLAGS) + +ifeq ($(cfg), release) + CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD +endif +ifeq ($(cfg), debug) + CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD +endif + +ifneq (,$(stdlib)) + CPLUS_FLAGS += -stdlib=$(stdlib) + LIB_LINK_FLAGS += -stdlib=$(stdlib) +endif + +TBB_ASM.OBJ= +MALLOC_ASM.OBJ= + +ifeq (intel64,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY + CPLUS_FLAGS += -m64 + LIB_LINK_FLAGS += -m64 +endif + +ifeq (ia32,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY + CPLUS_FLAGS += -m32 -march=pentium4 + LIB_LINK_FLAGS += -m32 +endif + +ifeq (ppc64,$(arch)) + CPLUS_FLAGS += -m64 + LIB_LINK_FLAGS += -m64 +endif + +ifeq (ppc32,$(arch)) + CPLUS_FLAGS += -m32 + LIB_LINK_FLAGS += -m32 +endif + +ifeq (bg,$(arch)) + CPLUS = bgclang++ + CONLY = bgclang +endif + +#------------------------------------------------------------------------------ +# Setting assembler data. +#------------------------------------------------------------------------------ +ASM = as +ifeq (intel64,$(arch)) + ASM_FLAGS += --64 +endif +ifeq (ia32,$(arch)) + ASM_FLAGS += --32 +endif +ifeq ($(cfg),debug) + ASM_FLAGS += -g +endif + +ASSEMBLY_SOURCE=$(arch)-gas +#------------------------------------------------------------------------------ +# End of setting assembler data. +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# Setting tbbmalloc data. +#------------------------------------------------------------------------------ + +M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions + +#------------------------------------------------------------------------------ +# End of setting tbbmalloc data. +#------------------------------------------------------------------------------ diff --git a/src/tbb/build/BSD.inc b/src/tbb/build/BSD.inc new file mode 100644 index 00000000..6ef75dba --- /dev/null +++ b/src/tbb/build/BSD.inc @@ -0,0 +1,70 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifndef arch + ifeq ($(shell uname -m),i386) + export arch:=ia32 + endif + ifeq ($(shell uname -m),ia64) + export arch:=ia64 + endif + ifeq ($(shell uname -m),amd64) + export arch:=intel64 + endif +endif + +ifndef runtime + clang_version:=$(shell clang -v 2>&1 | grep version | sed "s/.*version \([0-9]*\.[0-9]*\).*/\1/") + os_version:=$(shell uname -r) + os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') + export runtime:=cc$(clang_version)_kernel$(os_kernel_version) +endif + +native_compiler := clang +export compiler ?= clang +debugger ?= gdb + +CMD=$(SHELL) -c +CWD=$(shell pwd) +RM?=rm -f +RD?=rmdir +MD?=mkdir -p +NUL= /dev/null +SLASH=/ +MAKE_VERSIONS=sh $(tbb_root)/build/version_info_linux.sh $(VERSION_FLAGS) >version_string.ver +MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh + +ifdef LD_LIBRARY_PATH + export LD_LIBRARY_PATH := .:$(LD_LIBRARY_PATH) +else + export LD_LIBRARY_PATH := . +endif + +####### Build settings ######################################################## + +OBJ = o +DLL = so +LIBEXT=so + +TBB.LST = +TBB.DEF = +TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) +TBB.LIB = $(TBB.DLL) +LINK_TBB.LIB = $(TBB.LIB) + +MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) +MALLOC.LIB = $(MALLOC.DLL) +LINK_MALLOC.LIB = $(MALLOC.LIB) + +TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) diff --git a/src/tbb/build/FreeBSD.clang.inc b/src/tbb/build/FreeBSD.clang.inc index 3579603d..c7a2864d 100644 --- a/src/tbb/build/FreeBSD.clang.inc +++ b/src/tbb/build/FreeBSD.clang.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,100 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# - -COMPILE_ONLY = -c -MMD -PREPROC_ONLY = -E -x c++ -INCLUDE_KEY = -I -DEFINE_KEY = -D -OUTPUT_KEY = -o # -OUTPUTOBJ_KEY = -o # -PIC_KEY = -fPIC -WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else -DYLIB_KEY = -shared -EXPORT_KEY = -Wl,--version-script, -LIBDL = - -CPLUS = clang++ -CONLY = clang -LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) -LIBS += -lpthread -lrt -LINK_FLAGS = -Wl,-rpath-link=. -Wl,-rpath=. -rdynamic -C_FLAGS = $(CPLUS_FLAGS) - -ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD -endif -ifeq ($(cfg), debug) - CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD -endif - -ifneq (,$(stdlib)) - CPLUS_FLAGS += -stdlib=$(stdlib) - LIB_LINK_FLAGS += -stdlib=$(stdlib) -endif - -TBB_ASM.OBJ= -MALLOC_ASM.OBJ= - -ifeq (intel64,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ia32,$(arch)) - ITT_NOTIFY = -DDO_ITT_NOTIFY - CPLUS_FLAGS += -m32 -march=pentium4 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (ppc64,$(arch)) - CPLUS_FLAGS += -m64 - LIB_LINK_FLAGS += -m64 -endif - -ifeq (ppc32,$(arch)) - CPLUS_FLAGS += -m32 - LIB_LINK_FLAGS += -m32 -endif - -ifeq (bg,$(arch)) - CPLUS = bgclang++ - CONLY = bgclang -endif - -#------------------------------------------------------------------------------ -# Setting assembler data. -#------------------------------------------------------------------------------ -ASM = as -ifeq (intel64,$(arch)) - ASM_FLAGS += --64 -endif -ifeq (ia32,$(arch)) - ASM_FLAGS += --32 -endif -ifeq ($(cfg),debug) - ASM_FLAGS += -g -endif - -ASSEMBLY_SOURCE=$(arch)-gas -#------------------------------------------------------------------------------ -# End of setting assembler data. -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# Setting tbbmalloc data. -#------------------------------------------------------------------------------ -M_CPLUS_FLAGS = $(CPLUS_FLAGS) -fno-rtti -fno-exceptions +include $(tbb_root)/build/BSD.clang.inc -#------------------------------------------------------------------------------ -# End of setting tbbmalloc data. -#------------------------------------------------------------------------------ +LIBS += -lrt diff --git a/src/tbb/build/FreeBSD.gcc.inc b/src/tbb/build/FreeBSD.gcc.inc index 794cb7c4..653d4a3e 100644 --- a/src/tbb/build/FreeBSD.gcc.inc +++ b/src/tbb/build/FreeBSD.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ @@ -36,7 +32,7 @@ C_FLAGS = $(CPLUS_FLAGS) # gcc 6.0 and later have -flifetime-dse option that controls # elimination of stores done outside the object lifetime -ifneq (,$(shell gcc -dumpversion | egrep "^([6-9])")) +ifneq (,$(shell gcc -dumpfullversion -dumpversion | egrep "^([6-9])")) # keep pre-contruction stores for zero initialization DSE_KEY = -flifetime-dse=1 endif diff --git a/src/tbb/build/FreeBSD.inc b/src/tbb/build/FreeBSD.inc index 3bd6a064..7eafb274 100644 --- a/src/tbb/build/FreeBSD.inc +++ b/src/tbb/build/FreeBSD.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,64 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# - -ifndef arch - ifeq ($(shell uname -m),i386) - export arch:=ia32 - endif - ifeq ($(shell uname -m),ia64) - export arch:=ia64 - endif - ifeq ($(shell uname -m),amd64) - export arch:=intel64 - endif -endif - -ifndef runtime - clang_version:=$(shell clang -v 2>&1 | grep version | sed "s/.*version \([0-9]*\.[0-9]*\).*/\1/") - os_version:=$(shell uname -r) - os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') - export runtime:=cc$(clang_version)_kernel$(os_kernel_version) -endif - -native_compiler := clang -export compiler ?= clang -debugger ?= gdb - -CMD=$(SHELL) -c -CWD=$(shell pwd) -RM?=rm -f -RD?=rmdir -MD?=mkdir -p -NUL= /dev/null -SLASH=/ -MAKE_VERSIONS=sh $(tbb_root)/build/version_info_linux.sh $(VERSION_FLAGS) >version_string.ver -MAKE_TBBVARS=sh $(tbb_root)/build/generate_tbbvars.sh - -ifdef LD_LIBRARY_PATH - export LD_LIBRARY_PATH := .:$(LD_LIBRARY_PATH) -else - export LD_LIBRARY_PATH := . -endif - -####### Build settings ######################################################## - -OBJ = o -DLL = so -LIBEXT=so - -TBB.LST = -TBB.DEF = -TBB.DLL = libtbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) -TBB.LIB = $(TBB.DLL) -LINK_TBB.LIB = $(TBB.LIB) - -MALLOC.DLL = libtbbmalloc$(DEBUG_SUFFIX).$(DLL) -MALLOC.LIB = $(MALLOC.DLL) -LINK_MALLOC.LIB = $(MALLOC.LIB) -TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) +include $(tbb_root)/build/BSD.inc diff --git a/src/tbb/build/Makefile.rml b/src/tbb/build/Makefile.rml index 241d4bfb..62983a64 100644 --- a/src/tbb/build/Makefile.rml +++ b/src/tbb/build/Makefile.rml @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # TODO: investigate why version_string.ver is not complete when $(RML_SERVER.OBJ) is being compiled. .NOTPARALLEL: diff --git a/src/tbb/build/Makefile.tbb b/src/tbb/build/Makefile.tbb index b66c309b..8d155f80 100644 --- a/src/tbb/build/Makefile.tbb +++ b/src/tbb/build/Makefile.tbb @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# #------------------------------------------------------------------------------ # Define rules for making the TBB shared library. diff --git a/src/tbb/build/Makefile.tbbmalloc b/src/tbb/build/Makefile.tbbmalloc index dedcfa56..421e95c5 100644 --- a/src/tbb/build/Makefile.tbbmalloc +++ b/src/tbb/build/Makefile.tbbmalloc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # default target default_malloc: malloc malloc_test @@ -51,11 +47,11 @@ ORIG_LINK_MALLOC.LIB:=$(LINK_MALLOC.LIB) MALLOC_CPLUS.OBJ = backend.$(OBJ) large_objects.$(OBJ) backref.$(OBJ) tbbmalloc.$(OBJ) MALLOC.OBJ := $(MALLOC_CPLUS.OBJ) $(MALLOC_ASM.OBJ) itt_notify_malloc.$(OBJ) frontend.$(OBJ) PROXY.OBJ := proxy.$(OBJ) tbb_function_replacement.$(OBJ) -M_CPLUS_FLAGS := $(subst $(WARNING_KEY),,$(M_CPLUS_FLAGS)) $(DEFINE_KEY)__TBBMALLOC_BUILD=1 +M_CPLUS_FLAGS += $(DEFINE_KEY)__TBBMALLOC_BUILD=1 M_INCLUDES := $(INCLUDES) $(INCLUDE_KEY)$(MALLOC_ROOT) $(INCLUDE_KEY)$(MALLOC_SOURCE_ROOT) # Suppress superfluous warnings for TBBMalloc compilation -$(MALLOC.OBJ): M_CPLUS_FLAGS += $(WARNING_SUPPRESS) +$(MALLOC.OBJ): M_CPLUS_FLAGS := $(subst $(WARNING_KEY),,$(M_CPLUS_FLAGS)) $(WARNING_SUPPRESS) # Suppress superfluous warnings for TBBMalloc proxy compilation $(PROXY.OBJ): CPLUS_FLAGS += $(WARNING_SUPPRESS) @@ -138,11 +134,14 @@ MALLOC_TESTS = test_ScalableAllocator.$(TEST_EXT) \ test_malloc_pure_c.$(TEST_EXT) \ test_malloc_whitebox.$(TEST_EXT) \ test_malloc_used_by_lib.$(TEST_EXT) \ - test_malloc_lib_unload.$(TEST_EXT) + test_malloc_lib_unload.$(TEST_EXT) \ + test_malloc_shutdown_hang.$(TEST_EXT) ifneq (,$(MALLOCPROXY.DLL)) MALLOC_TESTS += test_malloc_overload.$(TEST_EXT) \ test_malloc_overload_proxy.$(TEST_EXT) \ - test_malloc_atexit.$(TEST_EXT) + test_malloc_overload_disable.$(TEST_EXT) \ + test_malloc_atexit.$(TEST_EXT) \ + test_malloc_new_handler.$(TEST_EXT) endif # ----------------------------------------------------- @@ -161,11 +160,14 @@ MALLOC_M_CPLUS_TESTS = test_malloc_whitebox.$(TEST_EXT) test_malloc_lib_unloa test_malloc_used_by_lib.$(TEST_EXT) MALLOC_NO_LIB_TESTS = test_malloc_whitebox.$(TEST_EXT) test_malloc_lib_unload.$(TEST_EXT) \ test_malloc_used_by_lib.$(TEST_EXT) test_malloc_overload.$(TEST_EXT) -MALLOC_LINK_PROXY_TESTS = test_malloc_overload_proxy.$(TEST_EXT) +MALLOC_LINK_PROXY_TESTS = test_malloc_overload_proxy.$(TEST_EXT) test_malloc_new_handler.$(TEST_EXT) MALLOC_ADD_DLL_TESTS = test_malloc_lib_unload.$(TEST_EXT) test_malloc_used_by_lib.$(TEST_EXT) \ test_malloc_atexit.$(TEST_EXT) +MALLOC_SUPPRESS_WARNINGS = test_malloc_whitebox.$(TEST_EXT) test_malloc_pure_c.$(TEST_EXT) -$(MALLOC_M_CPLUS_TESTS): CPLUS_FLAGS=$(M_CPLUS_FLAGS) +$(MALLOC_SUPPRESS_WARNINGS): WARNING_KEY= +$(MALLOC_SUPPRESS_WARNINGS): TEST_WARNING_KEY= +$(MALLOC_M_CPLUS_TESTS): CPLUS_FLAGS:=$(M_CPLUS_FLAGS) $(MALLOC_M_CPLUS_TESTS): INCLUDES=$(M_INCLUDES) $(MALLOC_NO_LIB_TESTS): LINK_MALLOC.LIB= $(MALLOC_NO_LIB_TESTS): LINK_FLAGS+=$(LIBDL) @@ -175,12 +177,12 @@ $(MALLOC_ADD_DLL_TESTS): %.$(TEST_EXT): %_dll.$(DLL) $(MALLOC_ADD_DLL_TESTS): TEST_LIBS+=$(@:.$(TEST_EXT)=_dll.$(LIBEXT)) endif -test_malloc_over%.$(TEST_EXT): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) +test_malloc_over%.$(TEST_EXT): CPLUS_FLAGS:=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) test_malloc_over%.$(TEST_EXT): INCLUDES=$(M_INCLUDES) test_malloc_overload_proxy.$(TEST_EXT): LINK_FLAGS+=$(LIBDL) -test_malloc_atexit_dll.$(DLL): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) -test_malloc_atexit.$(TEST_EXT): CPLUS_FLAGS=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) +test_malloc_atexit_dll.$(DLL): CPLUS_FLAGS:=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) +test_malloc_atexit.$(TEST_EXT): CPLUS_FLAGS:=$(subst /MT,/MD,$(M_CPLUS_FLAGS)) test_malloc_atexit.$(TEST_EXT): LINK_FLAGS+=$(LIBDL) # on Ubuntu 11.10 linker called with --as-needed, so dependency on libtbbmalloc_proxy # is not created, and malloc overload via linking with -ltbbmalloc_proxy is not working. @@ -205,9 +207,20 @@ test_malloc_whitebox.$(TEST_EXT): LINK_FILES+=$(MALLOC_ASM.OBJ) test_malloc_lib_unload_dll.$(DLL): CPLUS_FLAGS=$(ORIG_CPLUS_FLAGS) $(if $(no_exceptions),$(DEFINE_KEY)__TBB_TEST_NO_EXCEPTIONS=1) test_malloc_lib_unload_dll.$(DLL): INCLUDES=$(ORIG_INCLUDES) $(INCLUDE_TEST_HEADERS) -test_malloc_used_by_lib_dll.$(DLL): CPLUS_FLAGS=$(subst /MT,/LD,$(M_CPLUS_FLAGS)) +test_malloc_used_by_lib_dll.$(DLL): CPLUS_FLAGS:=$(subst /MT,/LD,$(M_CPLUS_FLAGS)) test_malloc_used_by_lib_dll.$(DLL): LINK_FILES+=$(ORIG_LINK_MALLOC.LIB) test_malloc_used_by_lib_dll.$(DLL): LIBDL= + +# The test needs both tbb and tbbmalloc. +# For static build LINK_TBB.LIB is resolved in tbb.a static lib name (Linux), which cannot be found (dynamic tbb is used only). +# In order to link properly, have to define LINK_TBB.LIB ourselves except for Windows where linkage with *.lib file expected. +ifdef extra_inc +ifneq ($(tbb_os),windows) +DYNAMIC_TBB_LIB=$(LIBPREF)tbb$(CPF_SUFFIX)$(DEBUG_SUFFIX).$(DLL) +endif +endif +test_malloc_shutdown_hang.$(TEST_EXT): LINK_FILES += $(if $(DYNAMIC_TBB_LIB), $(DYNAMIC_TBB_LIB), $(LINK_TBB.LIB)) + # ----------------------------------------------------- # ---- The list of TBBMalloc test running commands ---- @@ -220,6 +233,8 @@ ifneq (,$(MALLOCPROXY.DLL)) $(run_cmd) ./test_malloc_atexit.$(TEST_EXT) $(args) $(run_cmd) $(TEST_LAUNCHER) -l $(MALLOCPROXY.DLL) ./test_malloc_overload.$(TEST_EXT) $(args) $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_overload_proxy.$(TEST_EXT) $(args) + $(run_cmd) ./test_malloc_overload_disable.$(TEST_EXT) $(args) + $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_new_handler.$(TEST_EXT) $(args) endif $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_lib_unload.$(TEST_EXT) $(args) $(run_cmd) $(TEST_LAUNCHER) ./test_malloc_used_by_lib.$(TEST_EXT) @@ -230,6 +245,7 @@ endif $(run_cmd) ./test_malloc_regression.$(TEST_EXT) $(args) $(run_cmd) ./test_malloc_init_shutdown.$(TEST_EXT) $(args) $(run_cmd) ./test_malloc_pure_c.$(TEST_EXT) $(args) + $(run_cmd) ./test_malloc_shutdown_hang.$(TEST_EXT) # ----------------------------------------------------- #------------------------------------------------------ diff --git a/src/tbb/build/Makefile.tbbproxy b/src/tbb/build/Makefile.tbbproxy index eb2e0fd6..196361cc 100644 --- a/src/tbb/build/Makefile.tbbproxy +++ b/src/tbb/build/Makefile.tbbproxy @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # default target default_tbbproxy: tbbproxy tbbproxy_test diff --git a/src/tbb/build/Makefile.test b/src/tbb/build/Makefile.test index 9de7860d..dc801ab8 100644 --- a/src/tbb/build/Makefile.test +++ b/src/tbb/build/Makefile.test @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# #------------------------------------------------------------------------------ # Define rules for making the TBB tests. @@ -128,6 +124,7 @@ TEST_TBB_PLAIN.EXE = test_assembly.$(TEST_EXT) \ test_blocked_range.$(TEST_EXT) \ test_blocked_range2d.$(TEST_EXT) \ test_blocked_range3d.$(TEST_EXT) \ + test_blocked_rangeNd.$(TEST_EXT) \ test_concurrent_queue.$(TEST_EXT) \ test_concurrent_vector.$(TEST_EXT) \ test_concurrent_unordered_set.$(TEST_EXT) \ @@ -272,7 +269,7 @@ test_tbb_plain: $(TEST_PREREQUISITE) $(TESTS_TO_RUN) # For deprecated files, we don't mind warnings etc., thus compilation rules are most relaxed CPLUS_FLAGS_DEPRECATED = $(DEFINE_KEY)__TBB_TEST_DEPRECATED=1 $(subst $(WARNING_KEY),,$(CPLUS_FLAGS)) $(WARNING_SUPPRESS) $(INCLUDE_KEY)$(tbb_root)/src/test -TEST_TBB_OLD.OBJ = test_concurrent_vector_v2.$(OBJ) test_concurrent_queue_v2.$(OBJ) test_mutex_v2.$(OBJ) test_task_scheduler_observer_v3.$(OBJ) +TEST_TBB_OLD.OBJ = test_concurrent_vector_v2.$(OBJ) test_concurrent_queue_v2.$(OBJ) test_mutex_v2.$(OBJ) test_task_scheduler_observer_v3.$(OBJ) $(TEST_TBB_OLD.OBJ): CPLUS_FLAGS := $(CPLUS_FLAGS_DEPRECATED) diff --git a/src/tbb/build/OpenBSD.clang.inc b/src/tbb/build/OpenBSD.clang.inc new file mode 100644 index 00000000..dd913a37 --- /dev/null +++ b/src/tbb/build/OpenBSD.clang.inc @@ -0,0 +1,15 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include $(tbb_root)/build/BSD.clang.inc diff --git a/src/tbb/build/OpenBSD.inc b/src/tbb/build/OpenBSD.inc new file mode 100644 index 00000000..7eafb274 --- /dev/null +++ b/src/tbb/build/OpenBSD.inc @@ -0,0 +1,15 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include $(tbb_root)/build/BSD.inc diff --git a/src/tbb/build/SunOS.gcc.inc b/src/tbb/build/SunOS.gcc.inc index 2ebed62f..75cef4a8 100644 --- a/src/tbb/build/SunOS.gcc.inc +++ b/src/tbb/build/SunOS.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ @@ -25,8 +21,8 @@ OUTPUTOBJ_KEY = -o # PIC_KEY = -fPIC WARNING_AS_ERROR_KEY = -Werror WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor $(if $(findstring cc4., $(runtime)),-Wextra) -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor +TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -Wextra +WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor DYLIB_KEY = -shared LIBDL = -ldl diff --git a/src/tbb/build/SunOS.inc b/src/tbb/build/SunOS.inc index 447e059e..569d3214 100644 --- a/src/tbb/build/SunOS.inc +++ b/src/tbb/build/SunOS.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifndef arch arch:=$(shell uname -p) @@ -31,7 +27,7 @@ ifndef arch endif ifndef runtime - gcc_version:=$(shell gcc -dumpversion) + gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) diff --git a/src/tbb/build/SunOS.suncc.inc b/src/tbb/build/SunOS.suncc.inc index b9cc946e..db9d1a6a 100644 --- a/src/tbb/build/SunOS.suncc.inc +++ b/src/tbb/build/SunOS.suncc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -xMMD -errtags PREPROC_ONLY = -E -xMMD diff --git a/src/tbb/build/android.clang.inc b/src/tbb/build/android.clang.inc index a935968c..40b3149b 100644 --- a/src/tbb/build/android.clang.inc +++ b/src/tbb/build/android.clang.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD @@ -68,8 +64,20 @@ ifeq (0, $(dynamic_load)) endif # Paths to the NDK prebuilt tools and libraries -CPLUS_FLAGS += --sysroot=$(SYSROOT) +ifeq (,$(findstring $(ndk_version), $(foreach v, 7 8 9 10 11 12 13 14 15,r$(v) r$(v)b r$(v)c r$(v)d r$(v)e))) + # Since Android* NDK r16 another sysroot and isystem paths have to be specified + CPLUS_FLAGS += --sysroot=$(NDK_ROOT)/sysroot -isystem $(NDK_ROOT)/sysroot/usr/include/$(TRIPLE) + # Android* version flag required since r16 + CPLUS_FLAGS += -D__ANDROID_API__=$(API_LEVEL) +else + CPLUS_FLAGS += --sysroot=$(SYSROOT) +endif + +# Library sysroot flag LIB_LINK_FLAGS += --sysroot=$(SYSROOT) +# Flag for test executables +LINK_FLAGS += --sysroot=$(SYSROOT) + LIBS = -L$(CPLUS_LIB_PATH) -lc++_shared ifeq (,$(findstring $(ndk_version),$(foreach v, 7 8 9 10 11,r$(v) r$(v)b r$(v)c r$(v)d r$(v)e))) LIBS += -lc++abi diff --git a/src/tbb/build/android.gcc.inc b/src/tbb/build/android.gcc.inc index 973ca3f0..13b29fb8 100644 --- a/src/tbb/build/android.gcc.inc +++ b/src/tbb/build/android.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD diff --git a/src/tbb/build/android.icc.inc b/src/tbb/build/android.icc.inc index 64da9fd1..921a1f07 100644 --- a/src/tbb/build/android.icc.inc +++ b/src/tbb/build/android.icc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# COMPILE_ONLY = -c -MMD diff --git a/src/tbb/build/android.inc b/src/tbb/build/android.inc index aeeb68c5..893b33c1 100644 --- a/src/tbb/build/android.inc +++ b/src/tbb/build/android.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # # Extra gmake command-line parameters for use with Android: diff --git a/src/tbb/build/android.linux.inc b/src/tbb/build/android.linux.inc index 7d022434..39767b69 100644 --- a/src/tbb/build/android.linux.inc +++ b/src/tbb/build/android.linux.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### diff --git a/src/tbb/build/android.linux.launcher.sh b/src/tbb/build/android.linux.launcher.sh index b3cf8764..a394750c 100644 --- a/src/tbb/build/android.linux.launcher.sh +++ b/src/tbb/build/android.linux.launcher.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Usage: # android.linux.launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] diff --git a/src/tbb/build/android.macos.inc b/src/tbb/build/android.macos.inc index bf84578a..3efe09df 100644 --- a/src/tbb/build/android.macos.inc +++ b/src/tbb/build/android.macos.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### diff --git a/src/tbb/build/android.windows.inc b/src/tbb/build/android.windows.inc index f58e5d52..c690966f 100644 --- a/src/tbb/build/android.windows.inc +++ b/src/tbb/build/android.windows.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### diff --git a/src/tbb/build/big_iron.inc b/src/tbb/build/big_iron.inc index efed2125..dc8849f7 100644 --- a/src/tbb/build/big_iron.inc +++ b/src/tbb/build/big_iron.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# #------------------------------------------------------------------------------ # Defines settings for building the TBB run-time as a static library. diff --git a/src/tbb/build/build.py b/src/tbb/build/build.py new file mode 100644 index 00000000..4c3c1fb4 --- /dev/null +++ b/src/tbb/build/build.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Provides unified tool for preparing TBB for packaging + +from __future__ import print_function +import os +import re +import sys +import shutil +import platform +import argparse +from glob import glob +from collections import OrderedDict + +jp = os.path.join +is_win = (platform.system() == 'Windows') +is_lin = (platform.system() == 'Linux') +is_mac = (platform.system() == 'Darwin') + +default_prefix = os.getenv('PREFIX', 'install_prefix') +if is_win: + default_prefix = jp(default_prefix, 'Library') # conda-specific by default on Windows + +parser = argparse.ArgumentParser() +parser.add_argument('--tbbroot', default='.', help='Take Intel TBB from here') +parser.add_argument('--prefix', default=default_prefix, help='Prefix') +parser.add_argument('--prebuilt', default=[], action='append', help='Directories to find prebuilt files') +parser.add_argument('--no-rebuild', default=False, action='store_true', help='do not rebuild') +parser.add_argument('--install', default=False, action='store_true', help='install all') +parser.add_argument('--install-libs', default=False, action='store_true', help='install libs') +parser.add_argument('--install-devel', default=False, action='store_true', help='install devel') +parser.add_argument('--install-docs', default=False, action='store_true', help='install docs') +parser.add_argument('--install-python', default=False, action='store_true', help='install python module') +parser.add_argument('--make-tool', default='make', help='Use different make command instead') +parser.add_argument('--copy-tool', default=None, help='Use this command for copying ($ tool file dest-dir)') +parser.add_argument('--build-args', default="", help='specify extra build args') +parser.add_argument('--build-prefix', default='local', help='build dir prefix') +parser.add_argument('--cmake-dir', help='directory to install CMake configuration files. Default: /lib/cmake/tbb') +if is_win: + parser.add_argument('--msbuild', default=False, action='store_true', help='Use msbuild') + parser.add_argument('--vs', default="2012", help='select VS version for build') + parser.add_argument('--vs-platform', default="x64", help='select VS platform for build') +parser.add_argument('ignore', nargs='?', help="workaround conda-build issue #2512") + +args = parser.parse_args() + +if args.install: + args.install_libs = True + args.install_devel = True + args.install_docs = True + args.install_python= True + +def custom_cp(src, dst): + assert os.system(' '.join([args.copy_tool, src, dst])) == 0 + +if args.copy_tool: + install_cp = custom_cp # e.g. to use install -p -D -m 755 on Linux +else: + install_cp = shutil.copy + +bin_dir = jp(args.prefix, "bin") +lib_dir = jp(args.prefix, "lib") +inc_dir = jp(args.prefix, 'include') +doc_dir = jp(args.prefix, 'share', 'doc', 'tbb') +cmake_dir = jp(args.prefix, "lib", "cmake", "tbb") if args.cmake_dir is None else args.cmake_dir + +if is_win: + os.environ["OS"] = "Windows_NT" # make sure TBB will interpret it correctly + libext = '.dll' + libpref = '' + dll_dir = bin_dir +else: + libext = '.dylib' if is_mac else '.so.2' + libpref = 'lib' + dll_dir = lib_dir + +tbb_names = ["tbb", "tbbmalloc", "tbbmalloc_proxy"] + +############################################################## + +def system(arg): + print('$ ', arg) + return os.system(arg) + +def run_make(arg): + if system('%s -j %s'% (args.make_tool, arg)) != 0: + print("\nBummer. Running serial build in order to recover the log and have a chance to fix the build") + assert system('%s %s'% (args.make_tool, arg)) == 0 + +os.chdir(args.tbbroot) +if args.prebuilt: + release_dirs = sum([glob(d) for d in args.prebuilt], []) + print("Using pre-built files from ", release_dirs) +else: + if is_win and args.msbuild: + preview_release_dir = release_dir = jp(args.tbbroot, 'build', 'vs'+args.vs, args.vs_platform, 'Release') + if not args.no_rebuild or not os.path.isdir(release_dir): + assert os.system('msbuild /m /p:Platform=%s /p:Configuration=Release %s build/vs%s/makefile.sln'% \ + (args.vs_platform, args.build_args, args.vs)) == 0 + preview_debug_dir = debug_dir = jp(args.tbbroot, 'build', 'vs'+args.vs, args.vs_platform, 'Debug') + if not args.no_rebuild or not os.path.isdir(debug_dir): + assert os.system('msbuild /m /p:Platform=%s /p:Configuration=Debug %s build/vs%s/makefile.sln'% \ + (args.vs_platform, args.build_args, args.vs)) == 0 + else: + release_dir = jp(args.tbbroot, 'build', args.build_prefix+'_release') + debug_dir = jp(args.tbbroot, 'build', args.build_prefix+'_debug') + if not args.no_rebuild or not (os.path.isdir(release_dir) and os.path.isdir(debug_dir)): + run_make('tbb_build_prefix=%s %s'% (args.build_prefix, args.build_args)) + preview_release_dir = jp(args.tbbroot, 'build', args.build_prefix+'_preview_release') + preview_debug_dir = jp(args.tbbroot, 'build', args.build_prefix+'_preview_debug') + if not args.no_rebuild or not (os.path.isdir(preview_release_dir) and os.path.isdir(preview_debug_dir)): + run_make('tbb_build_prefix=%s_preview %s tbb_cpf=1 tbb'% (args.build_prefix, args.build_args)) + release_dirs = [release_dir, debug_dir, preview_release_dir, preview_debug_dir] + +filemap = OrderedDict() +def append_files(names, dst, paths=release_dirs): + global filemap + files = sum([glob(jp(d, f)) for d in paths for f in names], []) + filemap.update(dict(zip(files, [dst]*len(files)))) + + +if args.install_libs: + append_files([libpref+f+libext for f in tbb_names], dll_dir) + +if args.install_devel: + dll_files = [libpref+f+'_debug'+libext for f in tbb_names] # adding debug libraries + if not is_win or not args.msbuild: + dll_files += [libpref+"tbb_preview"+libext, libpref+"tbb_preview_debug"+libext] + if is_win: + dll_files += ['tbb*.pdb'] # copying debug info + if is_lin: + dll_files += ['libtbb*.so'] # copying linker scripts + # symlinks .so -> .so.2 should not be created instead + # since linking with -ltbb when using links can result in + # incorrect dependence upon unversioned .so files + append_files(dll_files, dll_dir) + if is_win: + append_files(['*.lib', '*.def'], lib_dir) # copying linker libs and defs + for rootdir, dirnames, filenames in os.walk(jp(args.tbbroot,'include')): + files = [f for f in filenames if not '.html' in f] + append_files(files, jp(inc_dir, rootdir.split('include')[1][1:]), paths=(rootdir,)) + + # Preparing CMake configuration files + cmake_build_dir = jp(args.tbbroot, 'build', args.build_prefix+'_release', 'cmake_configs') + assert system('cmake -DINSTALL_DIR=%s -DSYSTEM_NAME=%s -DTBB_VERSION_FILE=%s -DINC_REL_PATH=%s -DLIB_REL_PATH=%s -DBIN_REL_PATH=%s -P %s' % \ + (cmake_build_dir, + platform.system(), + jp(args.tbbroot, 'include', 'tbb', 'tbb_stddef.h'), + os.path.relpath(inc_dir, cmake_dir), + os.path.relpath(lib_dir, cmake_dir), + os.path.relpath(bin_dir, cmake_dir), + jp(args.tbbroot, 'cmake', 'tbb_config_installer.cmake'))) == 0 + append_files(['TBBConfig.cmake', 'TBBConfigVersion.cmake'], cmake_dir, paths=[cmake_build_dir]) + +if args.install_python: # RML part + irml_dir = jp(args.tbbroot, 'build', args.build_prefix+'_release') + run_make('-C src tbb_build_prefix=%s %s python_rml'% (args.build_prefix, args.build_args)) + if is_lin: + append_files(['libirml.so.1'], dll_dir, paths=[irml_dir]) + +if args.install_docs: + files = [ + 'CHANGES', + 'LICENSE', + 'README', + 'README.md', + 'Release_Notes.txt', + ] + append_files(files, doc_dir, paths=release_dirs+[jp(args.tbbroot, d) for d in ('.', 'doc')]) + +for f in filemap.keys(): + assert os.path.exists(f) + assert os.path.isfile(f) + +if filemap: + print("Copying to prefix =", args.prefix) +for f, dest in filemap.items(): + if not os.path.isdir(dest): + os.makedirs(dest) + print("+ %s to $prefix%s"%(f,dest.replace(args.prefix, ''))) + install_cp(f, dest) + +if args.install_python: # Python part + paths = [os.path.abspath(d) for d in [args.prefix, inc_dir, irml_dir, lib_dir]+release_dirs] + os.environ["TBBROOT"] = paths[0] + # all the paths must be relative to python/ directory or be absolute + assert system('python python/setup.py build -b%s build_ext -I%s -L%s install -f'% \ + (paths[2], paths[1], ':'.join(paths[2:]))) == 0 + +print("done") diff --git a/src/tbb/build/common.inc b/src/tbb/build/common.inc index b2c20de6..b95c78d6 100644 --- a/src/tbb/build/common.inc +++ b/src/tbb/build/common.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifndef tbb_os @@ -54,6 +50,14 @@ ifdef cpp0x override cpp0x= endif +# Define C & C++ compilers according to platform defaults or CXX & CC environment variables +ifneq (,$(findstring environment, $(origin CXX))) + CPLUS = $(CXX) +endif +ifneq (,$(findstring environment, $(origin CC))) + CONLY = $(CC) +endif + ifneq (,$(stdver)) ifeq (,$(findstring ++, $(stdver))) $(warning "Warning: unexpected stdver=$(stdver) is used.") @@ -152,8 +156,8 @@ ifndef BUILDING_PHASE .DELETE_ON_ERROR: # Make will delete target if error occurred when building it. -# MAKEOVERRIDES contains the command line variable definitions. Reseting it to -# empty allows propogating all exported overridden variables to nested makes. +# MAKEOVERRIDES contains the command line variable definitions. Resetting it to +# empty allows propagating all exported overridden variables to nested makes. # NOTEs: # 1. All variable set in command line are propagated to nested makes. # 2. All variables declared with the "export" keyword are propagated to diff --git a/src/tbb/build/common_rules.inc b/src/tbb/build/common_rules.inc index faf6546f..096a4075 100644 --- a/src/tbb/build/common_rules.inc +++ b/src/tbb/build/common_rules.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifeq ($(tbb_strict),1) @@ -63,6 +59,7 @@ ifeq ($(origin LIB_LINK_LIBS), undefined) LIB_LINK_LIBS = $(LIBDL) $(LIBS) endif +# some platforms do not provide separate C-only compiler CONLY ?= $(CPLUS) # The most generic rules @@ -88,7 +85,7 @@ $(foreach t,%.$(OBJ) $(TEST_AFFIXES_OBJS),$(eval $(call make-cxx-obj,$(t)))) #$(2) - is the input obj files and libraries define make-test-binary $(CPLUS) $(OUTPUT_KEY)$(strip $1) $(CPLUS_FLAGS) $(2) $(LIBS) $(LINK_FLAGS) -endef +endef # LINK_FILES the list of options to link test specific files (libraries and object files) LINK_FILES+=$(TEST_LIBS) diff --git a/src/tbb/build/detect.js b/src/tbb/build/detect.js index b2a0943b..ccbe9863 100644 --- a/src/tbb/build/detect.js +++ b/src/tbb/build/detect.js @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// function readAllFromFile(fname) { var fso = new ActiveXObject("Scripting.FileSystemObject"); @@ -55,7 +51,7 @@ function doWork() { WScript.Echo("unknown"); } } else { - tmpExec = WshShell.Exec(compilerPath + " -dumpversion"); + tmpExec = WshShell.Exec(compilerPath + " -dumpfullversion -dumpversion"); var gccVersion = tmpExec.StdOut.ReadLine(); if (WScript.Arguments(0) == "/runtime") { WScript.Echo("mingw" + gccVersion); diff --git a/src/tbb/build/generate_tbbvars.bat b/src/tbb/build/generate_tbbvars.bat index 8925f577..508f2748 100644 --- a/src/tbb/build/generate_tbbvars.bat +++ b/src/tbb/build/generate_tbbvars.bat @@ -1,66 +1,62 @@ -@echo off -REM -REM Copyright (c) 2005-2017 Intel Corporation -REM -REM Licensed under the Apache License, Version 2.0 (the "License"); -REM you may not use this file except in compliance with the License. -REM You may obtain a copy of the License at -REM -REM http://www.apache.org/licenses/LICENSE-2.0 -REM -REM Unless required by applicable law or agreed to in writing, software -REM distributed under the License is distributed on an "AS IS" BASIS, -REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -REM See the License for the specific language governing permissions and -REM limitations under the License. -REM -REM -REM -REM -REM -setlocal -for %%D in ("%tbb_root%") do set actual_root=%%~fD -set fslash_root=%actual_root:\=/% -set bin_dir=%CD% -set fslash_bin_dir=%bin_dir:\=/% -set _INCLUDE=INCLUDE& set _LIB=LIB -if not x%UNIXMODE%==x set _INCLUDE=CPATH& set _LIB=LIBRARY_PATH -if not x%USE_INCLUDE_ENV%==x set _INCLUDE=INCLUDE - -echo Generating local tbbvars.bat -echo @echo off>tbbvars.bat -echo SET TBBROOT=%actual_root%>>tbbvars.bat -echo SET TBB_ARCH_PLATFORM=%arch%\%runtime%>>tbbvars.bat -echo SET TBB_TARGET_ARCH=%arch%>>tbbvars.bat -echo SET %_INCLUDE%=%%TBBROOT%%\include;%%%_INCLUDE%%%>>tbbvars.bat -echo SET %_LIB%=%bin_dir%;%%%_LIB%%%>>tbbvars.bat -echo SET PATH=%bin_dir%;%%PATH%%>>tbbvars.bat -if not x%UNIXMODE%==x echo SET LD_LIBRARY_PATH=%bin_dir%;%%LD_LIBRARY_PATH%%>>tbbvars.bat - -echo Generating local tbbvars.sh -echo #!/bin/sh>tbbvars.sh -echo export TBBROOT="%fslash_root%">>tbbvars.sh -echo export TBB_ARCH_PLATFORM="%arch%\%runtime%">>tbbvars.sh -echo export TBB_TARGET_ARCH="%arch%">>tbbvars.sh -echo export %_INCLUDE%="${TBBROOT}/include;$%_INCLUDE%">>tbbvars.sh -echo export %_LIB%="%fslash_bin_dir%;$%_LIB%">>tbbvars.sh -echo export PATH="%fslash_bin_dir%;$PATH">>tbbvars.sh -if not x%UNIXMODE%==x echo export LD_LIBRARY_PATH="%fslash_bin_dir%;$LD_LIBRARY_PATH">>tbbvars.sh - -echo Generating local tbbvars.csh -echo #!/bin/csh>tbbvars.csh -echo setenv TBBROOT "%actual_root%">>tbbvars.csh -echo setenv TBB_ARCH_PLATFORM "%arch%\%runtime%">>tbbvars.csh -echo setenv TBB_TARGET_ARCH "%arch%">>tbbvars.csh -echo setenv %_INCLUDE% "${TBBROOT}\include;$%_INCLUDE%">>tbbvars.csh -echo setenv %_LIB% "%bin_dir%;$%_LIB%">>tbbvars.csh -echo setenv PATH "%bin_dir%;$PATH">>tbbvars.csh -if not x%UNIXMODE%==x echo setenv LD_LIBRARY_PATH "%bin_dir%;$LD_LIBRARY_PATH">>tbbvars.csh - -if not x%LIB_STL_ANDROID%==x ( -REM Workaround for copying Android* specific stl shared library to work folder -copy /Y "%LIB_STL_ANDROID:/=\%" . -) - -endlocal -exit +@echo off +REM +REM Copyright (c) 2005-2019 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM +setlocal +for %%D in ("%tbb_root%") do set actual_root=%%~fD +set fslash_root=%actual_root:\=/% +set bin_dir=%CD% +set fslash_bin_dir=%bin_dir:\=/% +set _INCLUDE=INCLUDE& set _LIB=LIB +if not x%UNIXMODE%==x set _INCLUDE=CPATH& set _LIB=LIBRARY_PATH +if not x%USE_INCLUDE_ENV%==x set _INCLUDE=INCLUDE + +echo Generating local tbbvars.bat +echo @echo off>tbbvars.bat +echo SET TBBROOT=%actual_root%>>tbbvars.bat +echo SET TBB_ARCH_PLATFORM=%arch%\%runtime%>>tbbvars.bat +echo SET TBB_TARGET_ARCH=%arch%>>tbbvars.bat +echo SET %_INCLUDE%=%%TBBROOT%%\include;%%%_INCLUDE%%%>>tbbvars.bat +echo SET %_LIB%=%bin_dir%;%%%_LIB%%%>>tbbvars.bat +echo SET PATH=%bin_dir%;%%PATH%%>>tbbvars.bat +if not x%UNIXMODE%==x echo SET LD_LIBRARY_PATH=%bin_dir%;%%LD_LIBRARY_PATH%%>>tbbvars.bat + +echo Generating local tbbvars.sh +echo #!/bin/sh>tbbvars.sh +echo export TBBROOT="%fslash_root%">>tbbvars.sh +echo export TBB_ARCH_PLATFORM="%arch%\%runtime%">>tbbvars.sh +echo export TBB_TARGET_ARCH="%arch%">>tbbvars.sh +echo export %_INCLUDE%="${TBBROOT}/include;$%_INCLUDE%">>tbbvars.sh +echo export %_LIB%="%fslash_bin_dir%;$%_LIB%">>tbbvars.sh +echo export PATH="%fslash_bin_dir%;$PATH">>tbbvars.sh +if not x%UNIXMODE%==x echo export LD_LIBRARY_PATH="%fslash_bin_dir%;$LD_LIBRARY_PATH">>tbbvars.sh + +echo Generating local tbbvars.csh +echo #!/bin/csh>tbbvars.csh +echo setenv TBBROOT "%actual_root%">>tbbvars.csh +echo setenv TBB_ARCH_PLATFORM "%arch%\%runtime%">>tbbvars.csh +echo setenv TBB_TARGET_ARCH "%arch%">>tbbvars.csh +echo setenv %_INCLUDE% "${TBBROOT}\include;$%_INCLUDE%">>tbbvars.csh +echo setenv %_LIB% "%bin_dir%;$%_LIB%">>tbbvars.csh +echo setenv PATH "%bin_dir%;$PATH">>tbbvars.csh +if not x%UNIXMODE%==x echo setenv LD_LIBRARY_PATH "%bin_dir%;$LD_LIBRARY_PATH">>tbbvars.csh + +if not x%LIB_STL_ANDROID%==x ( +REM Workaround for copying Android* specific stl shared library to work folder +copy /Y "%LIB_STL_ANDROID:/=\%" . +) + +endlocal +exit diff --git a/src/tbb/build/generate_tbbvars.sh b/src/tbb/build/generate_tbbvars.sh index 0ca0965a..49189f22 100644 --- a/src/tbb/build/generate_tbbvars.sh +++ b/src/tbb/build/generate_tbbvars.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate tbbvars.[c]sh scripts bin_dir="$PWD" # diff --git a/src/tbb/build/index.html b/src/tbb/build/index.html index 4d0ac7c8..ae047697 100644 --- a/src/tbb/build/index.html +++ b/src/tbb/build/index.html @@ -134,8 +134,8 @@

Software prerequisites:

Build and run as above, but additionally specify the version of the C++ standard or dialect to be used by the compiler. The specified value of stdver will be used as a parameter to the appropriate compiler option (such as -std); the behavior in case of unsupported value is compiler-specific. -
make target_app={win8ui, uwp} [target_mode=store] [(above options or targets)] -
Build and run as above, but use API that is compliant with Universal Windows* applications. Use win8ui option, if you want to use Intel TBB in Windows* 8 Universal application and uwp in case of Windows* 10 Universal Windows application. +
make target_app={win8ui, uwp, uwd} [target_mode=store] [(above options or targets)] +
Build and run as above, but use API that is compliant with Universal Windows* applications. Use win8ui option, if you want to use Intel TBB in Windows* 8 Universal application, uwp in case of Windows* 10 Universal Windows application and uwd for the usage inside Universal Windows* driver. target_mode=store is used to produce binaries that are compliant with Windows Store* application container. In later case they won't work with Intel TBB unit tests but work only with Windows Store* applications.
ndk-build target=android [(above options or targets)]
Build and run as above, but build libraries for Android* OS by Android NDK that should be installed. Makefiles were tested with revision 8. @@ -233,7 +233,7 @@

To port the Makefile infrastructure:

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel, the Intel logo and Itanium are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries.

diff --git a/src/tbb/build/ios.clang.inc b/src/tbb/build/ios.clang.inc index 265e9fa1..97629706 100644 --- a/src/tbb/build/ios.clang.inc +++ b/src/tbb/build/ios.clang.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# include $(tbb_root)/build/macos.clang.inc - -CPLUS_FLAGS += -miphoneos-version-min=8.0 -LINK_FLAGS += -miphoneos-version-min=8.0 -LIB_LINK_FLAGS += -miphoneos-version-min=8.0 diff --git a/src/tbb/build/ios.macos.inc b/src/tbb/build/ios.macos.inc index ae723bd5..684a82f7 100644 --- a/src/tbb/build/ios.macos.inc +++ b/src/tbb/build/ios.macos.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifneq ($(arch),$(filter $(arch),ia32 intel64 armv7 armv7s arm64)) $(error $(arch) is unknown architecture. Known arhitechtures are ia32 intel64 armv7 armv7s arm64) @@ -34,3 +30,5 @@ endif ios_version:=$(shell echo $(SDKROOT) | sed -e "s/.*[a-z,A-Z]\(.*\).sdk/\1/") runtime:=cc$(clang_version)_ios$(ios_version) + +IPHONEOS_DEPLOYMENT_TARGET ?= 8.0 diff --git a/src/tbb/build/linux.clang.inc b/src/tbb/build/linux.clang.inc index edabab26..14ac2ae3 100644 --- a/src/tbb/build/linux.clang.inc +++ b/src/tbb/build/linux.clang.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# +CPLUS ?= clang++ +CONLY ?= clang COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -24,22 +22,21 @@ OUTPUT_KEY = -o # OUTPUTOBJ_KEY = -o # PIC_KEY = -fPIC WARNING_AS_ERROR_KEY = -Werror -WARNING_KEY = -Wall -TEST_WARNING_KEY = -Wextra -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor +WARNING_KEY = -Wall -Wextra +TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor -Wno-dangling-else DYLIB_KEY = -shared EXPORT_KEY = -Wl,--version-script, LIBDL = -ldl -CPLUS = clang++ -CONLY = clang LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) LIBS += -lpthread -lrt LINK_FLAGS = -Wl,-rpath-link=. -rdynamic C_FLAGS = $(CPLUS_FLAGS) ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD +# CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD + CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -DUSE_PTHREAD endif ifeq ($(cfg), debug) CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD diff --git a/src/tbb/build/linux.gcc.inc b/src/tbb/build/linux.gcc.inc index 348bac0f..46324b19 100644 --- a/src/tbb/build/linux.gcc.inc +++ b/src/tbb/build/linux.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# +CPLUS ?= g++ +CONLY ?= gcc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -27,48 +25,48 @@ WARNING_AS_ERROR_KEY = -Werror WARNING_KEY = -Wall TEST_WARNING_KEY = -Wshadow -Wcast-qual -Woverloaded-virtual -Wnon-virtual-dtor -WARNING_SUPPRESS = -Wno-parentheses -Wno-non-virtual-dtor +WARNING_SUPPRESS = -Wno-parentheses DYLIB_KEY = -shared EXPORT_KEY = -Wl,--version-script, LIBDL = -ldl -CPLUS = g++ -CONLY = gcc LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) LIBS += -lpthread -lrt LINK_FLAGS = -Wl,-rpath-link=. -rdynamic C_FLAGS = $(CPLUS_FLAGS) # gcc 4.2 and higher support OpenMP -ifneq (,$(shell gcc -dumpversion | egrep "^(4\.[2-9]|[5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[2-9]|[5-9])")) OPENMP_FLAG = -fopenmp endif # gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them -ifneq (,$(shell gcc -dumpversion | egrep "^(4\.[8-9]|[5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[8-9]|[5-9])")) RTM_KEY = -mrtm endif # gcc 4.0 and later have -Wextra that is used by some our customers. -ifneq (,$(shell gcc -dumpversion | egrep "^([4-9])")) - TEST_WARNING_KEY += -Wextra +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([4-9])")) + WARNING_KEY += -Wextra endif -# gcc 5.0 and later have -Wsuggest-override option -# enable it via a pre-included header in order to limit to C++11 and above -ifneq (,$(shell gcc -dumpversion | egrep "^([5-9])")) +# gcc 5.0 and later have -Wsuggest-override and -Wno-sized-deallocation options +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([5-9])")) + # enable -Wsuggest-override via a pre-included header in order to limit to C++11 and above INCLUDE_TEST_HEADERS = -include $(tbb_root)/src/test/harness_preload.h + WARNING_SUPPRESS += -Wno-sized-deallocation endif # gcc 6.0 and later have -flifetime-dse option that controls # elimination of stores done outside the object lifetime -ifneq (,$(shell gcc -dumpversion | egrep "^([6-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([6-9])")) # keep pre-contruction stores for zero initialization DSE_KEY = -flifetime-dse=1 endif ifeq ($(cfg), release) - CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD +# CPLUS_FLAGS = $(ITT_NOTIFY) -g -O2 -DUSE_PTHREAD + CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -DUSE_PTHREAD endif ifeq ($(cfg), debug) CPLUS_FLAGS = -DTBB_USE_DEBUG $(ITT_NOTIFY) -g -O0 -DUSE_PTHREAD diff --git a/src/tbb/build/linux.icc.inc b/src/tbb/build/linux.icc.inc index c1830f57..4be0f02f 100644 --- a/src/tbb/build/linux.icc.inc +++ b/src/tbb/build/linux.icc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# +CPLUS ?= icpc +CONLY ?= icc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -33,13 +31,11 @@ ifneq (,$(shell icc -dumpversion | egrep "1[2-9]\.")) SDL_FLAGS = -fstack-protector -Wformat -Wformat-security endif -CPLUS = icpc -CONLY = icc - ITT_NOTIFY = -DDO_ITT_NOTIFY ifeq (release,$(cfg)) SDL_FLAGS += -D_FORTIFY_SOURCE=2 -CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -g -DUSE_PTHREAD +#CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -g -DUSE_PTHREAD +CPLUS_FLAGS = $(ITT_NOTIFY) -O2 -DUSE_PTHREAD else CPLUS_FLAGS = $(ITT_NOTIFY) -O0 -g -DUSE_PTHREAD -DTBB_USE_DEBUG endif diff --git a/src/tbb/build/linux.inc b/src/tbb/build/linux.inc index daeaaf9a..4d59aaac 100644 --- a/src/tbb/build/linux.inc +++ b/src/tbb/build/linux.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### @@ -59,7 +55,7 @@ ifndef arch endif ifndef runtime - export gcc_version:=$(shell gcc -dumpversion) + export gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export os_glibc_version_full:=$(shell getconf GNU_LIBC_VERSION | grep glibc | sed -e 's/^glibc //') diff --git a/src/tbb/build/linux.pathcc.inc b/src/tbb/build/linux.pathcc.inc index 948f2316..381b6360 100644 --- a/src/tbb/build/linux.pathcc.inc +++ b/src/tbb/build/linux.pathcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# +CPLUS ?= pathCC +CONLY ?= pathcc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -32,8 +30,6 @@ DYLIB_KEY = -shared EXPORT_KEY = -Wl,--version-script, LIBDL = -ldl -CPLUS = pathCC -CONLY = pathcc LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) LIBS += -lstl -lpthread -lrt LINK_FLAGS = -Wl,-rpath-link=. -rdynamic diff --git a/src/tbb/build/linux.xl.inc b/src/tbb/build/linux.xl.inc index fccb559f..165aab8a 100644 --- a/src/tbb/build/linux.xl.inc +++ b/src/tbb/build/linux.xl.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### +CPLUS ?= xlc++_r +CONLY ?= xlc_r COMPILE_ONLY = -c PREPROC_ONLY = -E -qsourcetype=c INCLUDE_KEY = -I @@ -34,8 +32,6 @@ DYLIB_KEY = -qmkshrobj EXPORT_KEY = -Wl,--version-script, LIBDL = -ldl -CPLUS = xlc++_r -CONLY = xlc_r LIB_LINK_FLAGS = $(DYLIB_KEY) -Wl,-soname=$(BUILDING_LIBRARY) LIBS = -lpthread -lrt C_FLAGS = $(CPLUS_FLAGS) diff --git a/src/tbb/build/macos.clang.inc b/src/tbb/build/macos.clang.inc index 32377053..a41f5c47 100644 --- a/src/tbb/build/macos.clang.inc +++ b/src/tbb/build/macos.clang.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# -CPLUS = clang++ -CONLY = clang +CPLUS ?= clang++ +CONLY ?= clang COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -39,15 +35,16 @@ LIB_LINK_FLAGS = -dynamiclib -install_name @rpath/$(BUILDING_LIBRARY) C_FLAGS = $(CPLUS_FLAGS) ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 +# CPLUS_FLAGS = -g -O2 + CPLUS_FLAGS = -O2 else CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG endif -CPLUS_FLAGS += -DUSE_PTHREAD +CPLUS_FLAGS += -DUSE_PTHREAD $(ITT_NOTIFY) # For Clang, we add the option to support RTM intrinsics *iff* xtest is found in -ifneq (,$(shell grep xtest `echo "\#include" | clang -E -M - 2>&1 | grep immintrin.h` 2>/dev/null)) +ifneq (,$(shell grep xtest `echo "\#include" | $(CONLY) -E -M - 2>&1 | grep immintrin.h` 2>/dev/null)) RTM_KEY = -mrtm endif @@ -57,12 +54,14 @@ ifneq (,$(stdlib)) endif ifeq (intel64,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY CPLUS_FLAGS += -m64 $(RTM_KEY) LINK_FLAGS += -m64 LIB_LINK_FLAGS += -m64 endif ifeq (ia32,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY CPLUS_FLAGS += -m32 $(RTM_KEY) LINK_FLAGS += -m32 LIB_LINK_FLAGS += -m32 @@ -92,6 +91,16 @@ ifdef SDKROOT LIB_LINK_FLAGS += -L$(SDKROOT)/usr/lib/system -L$(SDKROOT)/usr/lib/ endif +ifeq (ios,$(target)) + CPLUS_FLAGS += -miphoneos-version-min=$(IPHONEOS_DEPLOYMENT_TARGET) + LINK_FLAGS += -miphoneos-version-min=$(IPHONEOS_DEPLOYMENT_TARGET) + LIB_LINK_FLAGS += -miphoneos-version-min=$(IPHONEOS_DEPLOYMENT_TARGET) +else + CPLUS_FLAGS += -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) + LINK_FLAGS += -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) + LIB_LINK_FLAGS += -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) +endif + #------------------------------------------------------------------------------ # Setting assembler data. #------------------------------------------------------------------------------ diff --git a/src/tbb/build/macos.gcc.inc b/src/tbb/build/macos.gcc.inc index 65c7047c..140d128a 100644 --- a/src/tbb/build/macos.gcc.inc +++ b/src/tbb/build/macos.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# -CPLUS = g++ -CONLY = gcc +CPLUS ?= g++ +CONLY ?= gcc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -38,21 +34,42 @@ LINK_FLAGS = LIB_LINK_FLAGS = -dynamiclib -install_name @rpath/$(BUILDING_LIBRARY) C_FLAGS = $(CPLUS_FLAGS) +# gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[8-9]|[5-9])")) + RTM_KEY = -mrtm +endif + +# gcc 5.0 and later have -Wsuggest-override option +# enable it via a pre-included header in order to limit to C++11 and above +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([5-9])")) + INCLUDE_TEST_HEADERS = -include $(tbb_root)/src/test/harness_preload.h +endif + +# gcc 6.0 and later have -flifetime-dse option that controls +# elimination of stores done outside the object lifetime +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([6-9])")) + # keep pre-contruction stores for zero initialization + DSE_KEY = -flifetime-dse=1 +endif + ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 +# CPLUS_FLAGS = -g -O2 + CPLUS_FLAGS = -O2 else CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG endif -CPLUS_FLAGS += -DUSE_PTHREAD +CPLUS_FLAGS += -DUSE_PTHREAD $(ITT_NOTIFY) ifeq (intel64,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY CPLUS_FLAGS += -m64 LINK_FLAGS += -m64 LIB_LINK_FLAGS += -m64 endif ifeq (ia32,$(arch)) + ITT_NOTIFY = -DDO_ITT_NOTIFY CPLUS_FLAGS += -m32 LINK_FLAGS += -m32 LIB_LINK_FLAGS += -m32 diff --git a/src/tbb/build/macos.icc.inc b/src/tbb/build/macos.icc.inc index 92a339b2..5639eeec 100644 --- a/src/tbb/build/macos.icc.inc +++ b/src/tbb/build/macos.icc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# -CPLUS = icpc -CONLY = icc +CPLUS ?= icpc +CONLY ?= icc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -53,19 +49,20 @@ endif ifeq ($(cfg), release) SDL_FLAGS += -D_FORTIFY_SOURCE=2 - CPLUS_FLAGS = -g -O2 -fno-omit-frame-pointer +# CPLUS_FLAGS = -g -O2 -fno-omit-frame-pointer + CPLUS_FLAGS = -O2 -fno-omit-frame-pointer else CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG endif -CPLUS_FLAGS += -DUSE_PTHREAD +ITT_NOTIFY = -DDO_ITT_NOTIFY +CPLUS_FLAGS += -DUSE_PTHREAD $(ITT_NOTIFY) ifneq (,$(codecov)) CPLUS_FLAGS += -prof-gen=srcpos endif # ICC 14.0 and higher support usage of libc++, clang standard library -MACOSX_DEPLOYMENT_TARGET ?= 10.11 ifneq (,$(shell icc -dumpversion | egrep "^1[4-9]\.")) ifneq (,$(stdlib)) CPLUS_FLAGS += -stdlib=$(stdlib) -mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) diff --git a/src/tbb/build/macos.inc b/src/tbb/build/macos.inc index b22828d6..5011fc17 100644 --- a/src/tbb/build/macos.inc +++ b/src/tbb/build/macos.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ####### Detections and Commands ############################################### @@ -40,15 +36,15 @@ ifndef arch export arch:=ppc32 endif else - ifeq ($(shell /usr/sbin/sysctl -n hw.machine),arm64) - export arch:=arm64 + ifeq ($(shell /usr/sbin/sysctl -n hw.machine),arm64) + export arch:=arm64 + else + ifeq ($(shell /usr/sbin/sysctl -n hw.optional.x86_64 2>/dev/null),1) + export arch:=intel64 else - ifeq ($(shell /usr/sbin/sysctl -n hw.optional.x86_64 2>/dev/null),1) - export arch:=intel64 - else - export arch:=ia32 - endif + export arch:=ia32 endif + endif endif endif @@ -109,3 +105,5 @@ LINK_MALLOCPROXY.LIB = $(MALLOCPROXY.LIB) TEST_LAUNCHER=sh $(tbb_root)/build/test_launcher.sh $(largs) OPENCL.LIB = -framework OpenCL + +MACOSX_DEPLOYMENT_TARGET ?= 10.11 diff --git a/src/tbb/build/mic.icc.inc b/src/tbb/build/mic.icc.inc index 5574a8dc..39548d8b 100644 --- a/src/tbb/build/mic.icc.inc +++ b/src/tbb/build/mic.icc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# +CPLUS ?= icpc +CONLY ?= icc COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -30,8 +28,6 @@ EXPORT_KEY = -Wl,--version-script, NOINTRINSIC_KEY = -fno-builtin LIBDL = -ldl SDL_FLAGS = -fstack-protector -Wformat -Wformat-security -CPLUS = icpc -CONLY = icc ifeq (release,$(cfg)) SDL_FLAGS += -D_FORTIFY_SOURCE=2 diff --git a/src/tbb/build/mic.linux.inc b/src/tbb/build/mic.linux.inc index b739d617..4dcb5b02 100644 --- a/src/tbb/build/mic.linux.inc +++ b/src/tbb/build/mic.linux.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifeq ($(tbb_os),mic) $(error MIC supports only cross-compilation. Specify "target=mic" instead.) diff --git a/src/tbb/build/mic.linux.launcher.sh b/src/tbb/build/mic.linux.launcher.sh index ac69a43f..1376eb11 100644 --- a/src/tbb/build/mic.linux.launcher.sh +++ b/src/tbb/build/mic.linux.launcher.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Usage: # mic.linux.launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] diff --git a/src/tbb/build/mic.offload.inc b/src/tbb/build/mic.offload.inc index 96f0800f..407b9931 100644 --- a/src/tbb/build/mic.offload.inc +++ b/src/tbb/build/mic.offload.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifneq (mic,$(offload)) $(error File mic.offload.inc should not be included directly. Use offload=mic instead.) diff --git a/src/tbb/build/test_launcher.bat b/src/tbb/build/test_launcher.bat index 88c43dc4..d335db8b 100644 --- a/src/tbb/build/test_launcher.bat +++ b/src/tbb/build/test_launcher.bat @@ -1,74 +1,70 @@ -@echo off -REM -REM Copyright (c) 2005-2017 Intel Corporation -REM -REM Licensed under the Apache License, Version 2.0 (the "License"); -REM you may not use this file except in compliance with the License. -REM You may obtain a copy of the License at -REM -REM http://www.apache.org/licenses/LICENSE-2.0 -REM -REM Unless required by applicable law or agreed to in writing, software -REM distributed under the License is distributed on an "AS IS" BASIS, -REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -REM See the License for the specific language governing permissions and -REM limitations under the License. -REM -REM -REM -REM -REM - -set cmd_line= -if DEFINED run_prefix set cmd_line=%run_prefix% -:while -if NOT "%1"=="" ( - REM Verbose mode - if "%1"=="-v" ( - set verbose=yes - GOTO continue - ) - REM Silent mode of 'make' requires additional support for associating - REM of test output with the test name. Verbose mode is the simplest way - if "%1"=="-q" ( - set verbose=yes - GOTO continue - ) - REM Run in stress mode - if "%1"=="-s" ( - echo Doing stress testing. Press Ctrl-C to terminate - set stress=yes - GOTO continue - ) - REM Repeat execution specified number of times - if "%1"=="-r" ( - set repeat=%2 - SHIFT - GOTO continue - ) - REM no LD_PRELOAD under Windows - REM but run the test to check "#pragma comment" construction - if "%1"=="-l" ( - REM The command line may specify -l with empty dll name, - REM e.g. "test_launcher.bat -l app.exe". If the dll name is - REM empty then %2 contains the application name and the SHIFT - REM operation is not necessary. - if exist "%3" SHIFT - GOTO continue - ) - REM no need to setup up stack size under Windows - if "%1"=="-u" GOTO continue - set cmd_line=%cmd_line% %1 -:continue - SHIFT - GOTO while -) -set cmd_line=%cmd_line:./=.\% -if DEFINED verbose echo Running %cmd_line% -if DEFINED stress set cmd_line=%cmd_line% ^& IF NOT ERRORLEVEL 1 GOTO stress -:stress -if DEFINED repeat ( - for /L %%i in (1,1,%repeat%) do echo %%i of %repeat%: & %cmd_line% -) else ( - %cmd_line% -) +@echo off +REM +REM Copyright (c) 2005-2019 Intel Corporation +REM +REM Licensed under the Apache License, Version 2.0 (the "License"); +REM you may not use this file except in compliance with the License. +REM You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +set cmd_line= +if DEFINED run_prefix set cmd_line=%run_prefix% +:while +if NOT "%1"=="" ( + REM Verbose mode + if "%1"=="-v" ( + set verbose=yes + GOTO continue + ) + REM Silent mode of 'make' requires additional support for associating + REM of test output with the test name. Verbose mode is the simplest way + if "%1"=="-q" ( + set verbose=yes + GOTO continue + ) + REM Run in stress mode + if "%1"=="-s" ( + echo Doing stress testing. Press Ctrl-C to terminate + set stress=yes + GOTO continue + ) + REM Repeat execution specified number of times + if "%1"=="-r" ( + set repeat=%2 + SHIFT + GOTO continue + ) + REM no LD_PRELOAD under Windows + REM but run the test to check "#pragma comment" construction + if "%1"=="-l" ( + REM The command line may specify -l with empty dll name, + REM e.g. "test_launcher.bat -l app.exe". If the dll name is + REM empty then %2 contains the application name and the SHIFT + REM operation is not necessary. + if exist "%3" SHIFT + GOTO continue + ) + REM no need to setup up stack size under Windows + if "%1"=="-u" GOTO continue + set cmd_line=%cmd_line% %1 +:continue + SHIFT + GOTO while +) +set cmd_line=%cmd_line:./=.\% +if DEFINED verbose echo Running %cmd_line% +if DEFINED stress set cmd_line=%cmd_line% ^& IF NOT ERRORLEVEL 1 GOTO stress +:stress +if DEFINED repeat ( + for /L %%i in (1,1,%repeat%) do echo %%i of %repeat%: & %cmd_line% +) else ( + %cmd_line% +) diff --git a/src/tbb/build/test_launcher.sh b/src/tbb/build/test_launcher.sh index 831136f0..7cb81958 100644 --- a/src/tbb/build/test_launcher.sh +++ b/src/tbb/build/test_launcher.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Usage: # test_launcher.sh [-v] [-q] [-s] [-r ] [-u] [-l ] diff --git a/src/tbb/build/version_info_aix.sh b/src/tbb/build/version_info_aix.sh index 622a22ed..c59ed497 100644 --- a/src/tbb/build/version_info_aix.sh +++ b/src/tbb/build/version_info_aix.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate version info string echo "#define __TBB_VERSION_STRINGS(N) \\" diff --git a/src/tbb/build/version_info_android.sh b/src/tbb/build/version_info_android.sh index 2439207f..3bef6039 100644 --- a/src/tbb/build/version_info_android.sh +++ b/src/tbb/build/version_info_android.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate version info string echo "#define __TBB_VERSION_STRINGS(N) \\" diff --git a/src/tbb/build/version_info_linux.sh b/src/tbb/build/version_info_linux.sh index 622a22ed..c59ed497 100644 --- a/src/tbb/build/version_info_linux.sh +++ b/src/tbb/build/version_info_linux.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate version info string echo "#define __TBB_VERSION_STRINGS(N) \\" diff --git a/src/tbb/build/version_info_macos.sh b/src/tbb/build/version_info_macos.sh index 397b1081..d49fcf8e 100644 --- a/src/tbb/build/version_info_macos.sh +++ b/src/tbb/build/version_info_macos.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate version info string echo "#define __TBB_VERSION_STRINGS(N) \\" diff --git a/src/tbb/build/version_info_sunos.sh b/src/tbb/build/version_info_sunos.sh index 96cd9ba7..4e54169b 100644 --- a/src/tbb/build/version_info_sunos.sh +++ b/src/tbb/build/version_info_sunos.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Script used to generate version info string echo "#define __TBB_VERSION_STRINGS(N) \\" diff --git a/src/tbb/build/version_info_windows.js b/src/tbb/build/version_info_windows.js index 57483b3c..223650ab 100644 --- a/src/tbb/build/version_info_windows.js +++ b/src/tbb/build/version_info_windows.js @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// var WshShell = WScript.CreateObject("WScript.Shell"); diff --git a/src/tbb/build/vs2013/index.html b/src/tbb/build/vs2013/index.html index 4e597b66..40b85f34 100644 --- a/src/tbb/build/vs2013/index.html +++ b/src/tbb/build/vs2013/index.html @@ -20,7 +20,7 @@

Files

Up to parent directory

-Copyright © 2017 Intel Corporation. All Rights Reserved. +Copyright © 2017-2019 Intel Corporation. All Rights Reserved.

Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries. diff --git a/src/tbb/build/vs2013/makefile.sln b/src/tbb/build/vs2013/makefile.sln index b913551e..6f03fe3d 100644 --- a/src/tbb/build/vs2013/makefile.sln +++ b/src/tbb/build/vs2013/makefile.sln @@ -1,80 +1,80 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8898CE0B-0BFB-45AE-AA71-83735ED2510D}" - ProjectSection(SolutionItems) = preProject - index.html = index.html - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbb", "tbb.vcxproj", "{F62787DD-1327-448B-9818-030062BCFAA5}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc", "tbbmalloc.vcxproj", "{B15F131E-328A-4D42-ADC2-9FF4CA6306D8}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc_proxy", "tbbmalloc_proxy.vcxproj", "{02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Debug-MT|Win32 = Debug-MT|Win32 - Debug-MT|x64 = Debug-MT|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - Release-MT|Win32 = Release-MT|Win32 - Release-MT|x64 = Release-MT|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.ActiveCfg = Debug|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.Build.0 = Debug|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.ActiveCfg = Debug|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.Build.0 = Debug|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.ActiveCfg = Release|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.Build.0 = Release|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.ActiveCfg = Release|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.Build.0 = Release|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.Build.0 = Release-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.ActiveCfg = Debug|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.Build.0 = Debug|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.ActiveCfg = Debug|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.Build.0 = Debug|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.ActiveCfg = Release|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.Build.0 = Release|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.ActiveCfg = Release|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.Build.0 = Release|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.Build.0 = Release-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.ActiveCfg = Debug|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.Build.0 = Debug|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.ActiveCfg = Debug|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.Build.0 = Debug|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.Build.0 = Debug-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.ActiveCfg = Release|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.Build.0 = Release|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.ActiveCfg = Release|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.Build.0 = Release|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.Build.0 = Release-MT|Win32 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.ActiveCfg = Release-MT|x64 - {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.Build.0 = Release-MT|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.40629.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8898CE0B-0BFB-45AE-AA71-83735ED2510D}" + ProjectSection(SolutionItems) = preProject + index.html = index.html + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbb", "tbb.vcxproj", "{F62787DD-1327-448B-9818-030062BCFAA5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc", "tbbmalloc.vcxproj", "{B15F131E-328A-4D42-ADC2-9FF4CA6306D8}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tbbmalloc_proxy", "tbbmalloc_proxy.vcxproj", "{02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Debug-MT|Win32 = Debug-MT|Win32 + Debug-MT|x64 = Debug-MT|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + Release-MT|Win32 = Release-MT|Win32 + Release-MT|x64 = Release-MT|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.ActiveCfg = Debug|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|Win32.Build.0 = Debug|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.ActiveCfg = Debug|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug|x64.Build.0 = Debug|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Debug-MT|x64.Build.0 = Debug-MT|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.ActiveCfg = Release|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release|Win32.Build.0 = Release|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.ActiveCfg = Release|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release|x64.Build.0 = Release|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|Win32.Build.0 = Release-MT|Win32 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.ActiveCfg = Release-MT|x64 + {F62787DD-1327-448B-9818-030062BCFAA5}.Release-MT|x64.Build.0 = Release-MT|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.ActiveCfg = Debug|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|Win32.Build.0 = Debug|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.ActiveCfg = Debug|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug|x64.Build.0 = Debug|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Debug-MT|x64.Build.0 = Debug-MT|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.ActiveCfg = Release|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|Win32.Build.0 = Release|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.ActiveCfg = Release|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release|x64.Build.0 = Release|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|Win32.Build.0 = Release-MT|Win32 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.ActiveCfg = Release-MT|x64 + {B15F131E-328A-4D42-ADC2-9FF4CA6306D8}.Release-MT|x64.Build.0 = Release-MT|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.ActiveCfg = Debug|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|Win32.Build.0 = Debug|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.ActiveCfg = Debug|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug|x64.Build.0 = Debug|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.ActiveCfg = Debug-MT|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|Win32.Build.0 = Debug-MT|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.ActiveCfg = Debug-MT|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Debug-MT|x64.Build.0 = Debug-MT|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.ActiveCfg = Release|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|Win32.Build.0 = Release|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.ActiveCfg = Release|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release|x64.Build.0 = Release|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.ActiveCfg = Release-MT|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|Win32.Build.0 = Release-MT|Win32 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.ActiveCfg = Release-MT|x64 + {02F61511-D5B6-46E6-B4BB-DEAA96E6BCC7}.Release-MT|x64.Build.0 = Release-MT|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/tbb/build/vs2013/tbb.vcxproj b/src/tbb/build/vs2013/tbb.vcxproj index 7329d81b..cbfcd391 100644 --- a/src/tbb/build/vs2013/tbb.vcxproj +++ b/src/tbb/build/vs2013/tbb.vcxproj @@ -599,6 +599,7 @@ + diff --git a/src/tbb/build/vs2013/tbbmalloc.vcxproj b/src/tbb/build/vs2013/tbbmalloc.vcxproj index 90d5fe13..0a38cd5f 100644 --- a/src/tbb/build/vs2013/tbbmalloc.vcxproj +++ b/src/tbb/build/vs2013/tbbmalloc.vcxproj @@ -482,6 +482,7 @@ + diff --git a/src/tbb/build/windows.cl.inc b/src/tbb/build/windows.cl.inc index 3906d086..b937692c 100644 --- a/src/tbb/build/windows.cl.inc +++ b/src/tbb/build/windows.cl.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# #------------------------------------------------------------------------------ # Define compiler-specific variables. @@ -24,7 +20,7 @@ #------------------------------------------------------------------------------ # Setting compiler flags. #------------------------------------------------------------------------------ -CPLUS = cl /nologo +CPLUS ?= cl /nologo LINK_FLAGS = /link /nologo LIB_LINK_FLAGS=/link /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DYNAMICBASE /NXCOMPAT @@ -43,6 +39,11 @@ else endif EH_FLAGS = $(if $(no_exceptions),/EHs-,/EHsc /GR) +# UWD binaries have to use static CRT linkage +ifeq ($(target_app), uwd) + MS_CRT_KEY = /MT$(if $(findstring debug,$(cfg)),d) +endif + ifeq ($(cfg), release) CPLUS_FLAGS = $(MS_CRT_KEY) /O2 /Zi $(EH_FLAGS) /Zc:forScope /Zc:wchar_t /D__TBB_LIB_NAME=$(TBB.LIB) ASM_FLAGS = @@ -54,17 +55,20 @@ endif ZW_KEY = /ZW:nostdlib -ifneq (,$(filter win8ui,$(target_app) $(target_ui))) - CPLUS_FLAGS += $(ZW_KEY) /D "_UNICODE" /D "UNICODE" /D "WINAPI_FAMILY=WINAPI_FAMILY_APP" +# These flags are general for Windows* universal applications +ifneq (,$(target_app)) + CPLUS_FLAGS += $(ZW_KEY) /D "_UNICODE" /D "UNICODE" /D "WINAPI_FAMILY=WINAPI_FAMILY_APP" +endif + +ifeq ($(target_app), win8ui) _WIN32_WINNT = 0x0602 -else ifneq (,$(filter uwp,$(target_app) $(target_ui))) - CPLUS_FLAGS += $(ZW_KEY) /D "_UNICODE" /D "UNICODE" /D "WINAPI_FAMILY=WINAPI_FAMILY_APP" +else ifneq (,$(filter $(target_app),uwp uwd)) _WIN32_WINNT = 0x0A00 LIB_LINK_FLAGS += /NODEFAULTLIB:kernel32.lib OneCore.lib else CPLUS_FLAGS += /DDO_ITT_NOTIFY endif -ifneq (,$(filter store,$(target_mode) $(target_ui_mode))) +ifeq ($(target_mode), store) # it is necessary to source vcvars with 'store' argument in production LIB_LINK_FLAGS += /APPCONTAINER endif diff --git a/src/tbb/build/windows.gcc.inc b/src/tbb/build/windows.gcc.inc index e4dad35c..2dfb081c 100644 --- a/src/tbb/build/windows.gcc.inc +++ b/src/tbb/build/windows.gcc.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# # Set by R R_COMPILED_BY ?= 4.9.3 @@ -53,7 +49,7 @@ PROXY.LIB = # Compiler-specific variables #------------------------------------------------------------------------------ -# CPLUS = g++ +# CPLUS ?= g++ COMPILE_ONLY = -c -MMD PREPROC_ONLY = -E -x c++ INCLUDE_KEY = -I @@ -93,7 +89,7 @@ ifeq ($(shell expr $(GCC_VERSION) \>= 6.0), 1) endif ifeq ($(cfg), release) - CPLUS_FLAGS = -g -O2 + CPLUS_FLAGS = -O2 endif ifeq ($(cfg), debug) CPLUS_FLAGS = -g -O0 -DTBB_USE_DEBUG diff --git a/src/tbb/build/windows.icl.inc b/src/tbb/build/windows.icl.inc index 7a0b1165..8010d5fc 100644 --- a/src/tbb/build/windows.icl.inc +++ b/src/tbb/build/windows.icl.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# #------------------------------------------------------------------------------ # Define compiler-specific variables. @@ -33,7 +29,7 @@ cfg ?= release #------------------------------------------------------------------------------ # Setting compiler flags. #------------------------------------------------------------------------------ -CPLUS = icl /nologo $(VCCOMPAT_FLAG) +CPLUS ?= icl /nologo $(VCCOMPAT_FLAG) LINK_FLAGS = /link /nologo LIB_LINK_FLAGS= /link /nologo /DLL /MAP /DEBUG /fixed:no /INCREMENTAL:NO /DYNAMICBASE /NXCOMPAT diff --git a/src/tbb/build/windows.inc b/src/tbb/build/windows.inc index ea90a3a6..01bccfdd 100644 --- a/src/tbb/build/windows.inc +++ b/src/tbb/build/windows.inc @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# ifdef tbb_build_dir test_dir:=$(tbb_build_dir) @@ -126,7 +122,7 @@ MAKE_VERSIONS = $(CMD) cscript //NoLogo //E:JScript \ "$(arch)" \ "$(subst \,/,$(VERSION_FLAGS))" \ > version_string.ver - + MAKE_TBBVARS = $(CMD) "$(subst /,\,$(tbb_root)/build/generate_tbbvars.bat)" TEST_LAUNCHER = $(subst /,\,$(tbb_root))\build\test_launcher.bat $(largs) diff --git a/src/tbb/cmake/README.rst b/src/tbb/cmake/README.rst new file mode 100644 index 00000000..3b16c37c --- /dev/null +++ b/src/tbb/cmake/README.rst @@ -0,0 +1,360 @@ +.. contents:: + +Introduction +------------ +Many developers use CMake to manage their development projects, so the Threading Building Blocks (TBB) +team created the set of CMake modules to simplify integration of the TBB library into a CMake project. +The modules are available starting from TBB 2017 U7 in `/cmake `_. + +About TBB +^^^^^^^^^^^^^^^ +TBB is a library that supports scalable parallel programming using standard ISO C++ code. It does not require special languages or compilers. It is designed to promote scalable data parallel programming. Additionally, it fully supports nested parallelism, so you can build larger parallel components from smaller parallel components. To use the library, you specify tasks, not threads, and let the library map tasks onto threads in an efficient manner. + +Many of the library interfaces employ generic programming, in which interfaces are defined by requirements on types and not specific types. The C++ Standard Template Library (STL) is an example of generic programming. Generic programming enables TBB to be flexible yet efficient. The generic interfaces enable you to customize components to your specific needs. + +The net result is that TBB enables you to specify parallelism far more conveniently than using raw threads, and at the same time can improve performance. + +References +^^^^^^^^^^ +* `Official TBB open source site `_ +* `Official GitHub repository `_ + +Engineering team contacts +^^^^^^^^^^^^^^^^^^^^^^^^^ +The TBB team is very interested in convenient integration of the TBB library into customer projects. These CMake modules were created to provide such a possibility for CMake projects using a simple but powerful interface. We hope you will try these modules and we are looking forward to receiving your feedback! + +E-mail us: `inteltbbdevelopers@intel.com `_. + +Visit our `forum `_. + +Release Notes +------------- +* Minimum supported CMake version: ``3.0.0``. +* TBB versioning via `find_package `_ has the following format: ``find_package(TBB .. ...)``. TBB interface version can also be obtained in the customer project via the ``TBB_INTERFACE_VERSION`` variable. + +Use cases of TBB integration into CMake-aware projects +------------------------------------------------------------ +There are two types of TBB packages: + * Binary packages with pre-built binaries for Windows* OS, Linux* OS and macOS*. They are available on the releases page of the Github repository: https://github.com/01org/tbb/releases. The main purpose of the binary package integration is the ability to build TBB header files and binaries into your CMake-aware project. + * A source package is also available to download from the release page via the "Source code" link. In addition, it can be cloned from the repository by ``git clone https://github.com/01org/tbb.git``. The main purpose of the source package integration is to allow you to do a custom build of the TBB library from the source files and then build that into your CMake-aware project. + +There are four types of CMake modules that can be used to integrate TBB: `TBBConfig`, `TBBGet`, `TBBMakeConfig` and `TBBBuild`. See `Technical documentation for CMake modules`_ section for additional details. + +Binary package integration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following use case is valid for packages starting from TBB 2017 U7: + +* Download package manually and make integration. + + Pre-condition: Location of TBBConfig.cmake is available via ``TBB_DIR`` or ``CMAKE_PREFIX_PATH`` contains path to TBB root. + + CMake code for integration: + .. code:: cmake + + find_package(TBB ) + +The following use case is valid for all TBB 2017 packages. + +* Download package using TBBGet_ and make integration. + + Pre-condition: TBB CMake modules are available via . + + CMake code for integration: + .. code:: cmake + + include(/TBBGet.cmake) + tbb_get(TBB_ROOT tbb_root CONFIG_DIR TBB_DIR) + find_package(TBB ) + +Source package integration +^^^^^^^^^^^^^^^^^^^^^^^^^^ +* Build TBB from existing source files using TBBBuild_ and make integration. + + Pre-condition: TBB source code is available via and TBB CMake modules are available via . + + CMake code for integration: + .. code:: cmake + + include(/TBBBuild.cmake) + tbb_build(TBB_ROOT CONFIG_DIR TBB_DIR) + find_package(TBB ) + +* Download TBB source files using TBBGet_, build it using TBBBuild_ and make integration. + + Pre-condition: TBB CMake modules are available via . + + CMake code for integration: + .. code:: cmake + + include(/TBBGet.cmake) + include(/TBBBuild.cmake) + tbb_get(TBB_ROOT tbb_root SOURCE_CODE) + tbb_build(TBB_ROOT ${tbb_root} CONFIG_DIR TBB_DIR) + find_package(TBB ) + +Tutorials: TBB integration using CMake +-------------------------------------------- +Binary TBB integration to the sub_string_finder sample (Windows* OS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we will integrate binary TBB package into the sub_string_finder sample on Windows* OS (Microsoft* Visual Studio). +This example is also applicable for other platforms with slight changes. +Place holders and should be replaced with the actual values for the TBB package being used. The example is written for `CMake 3.7.1`. + +Precondition: + * `Microsoft* Visual Studio 11` or higher. + * `CMake 3.0.0` or higher. + +#. Download the latest binary package for Windows from `this page `_ and unpack it to the directory ``C:\demo_tbb_cmake``. +#. In the directory ``C:\demo_tbb_cmake\tbb_oss\examples\GettingStarted\sub_string_finder`` create ``CMakeLists.txt`` file with the following content: + .. code:: cmake + + cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR) + + project(sub_string_finder CXX) + add_executable(sub_string_finder sub_string_finder.cpp) + + # find_package will search for available TBBConfig using variables CMAKE_PREFIX_PATH and TBB_DIR. + find_package(TBB REQUIRED tbb) + + # Link TBB imported targets to the executable; + # "TBB::tbb" can be used instead of "${TBB_IMPORTED_TARGETS}". + target_link_libraries(sub_string_finder ${TBB_IMPORTED_TARGETS}) +#. Run CMake GUI and: + * Fill the following fields (you can use the buttons ``Browse Source...`` and ``Browse Build...`` accordingly) + + * Where is the source code: ``C:/demo_tbb_cmake/tbb_oss/examples/GettingStarted/sub_string_finder`` + * Where to build the binaries: ``C:/demo_tbb_cmake/tbb_oss/examples/GettingStarted/sub_string_finder/build`` + + * Add new cache entry using button ``Add Entry`` to let CMake know where to search for TBBConfig: + + * Name: ``CMAKE_PREFIX_PATH`` + * Type: ``PATH`` + * Value: ``C:/demo_tbb_cmake/tbb_oss`` + + * Push the button ``Generate`` and choose a proper generator for your Microsoft* Visual Studio version. +#. Now you can open the generated solution ``C:/demo_tbb_cmake/tbb_oss/examples/GettingStarted/sub_string_finder/build/sub_string_finder.sln`` in your Microsoft* Visual Studio and build it. + +Source code integration of TBB to the sub_string_finder sample (Linux* OS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we will build TBB from source code with enabled Community Preview Features and link the sub_string_finder sample with the built library. +This example is also applicable for other platforms with slight changes. + +Precondition: + * `CMake 3.0.0` or higher. + * `Git` (to clone the TBB repository from GitHub) + +#. Create the directory ``~/demo_tbb_cmake``, go to the created directory and clone the TBB repository there: + ``mkdir ~/demo_tbb_cmake ; cd ~/demo_tbb_cmake ; git clone https://github.com/01org/tbb.git`` +#. In the directory ``~/demo_tbb_cmake/tbb/examples/GettingStarted/sub_string_finder`` create ``CMakeLists.txt`` file with following content: + .. code:: cmake + + cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR) + + project(sub_string_finder CXX) + add_executable(sub_string_finder sub_string_finder.cpp) + + include(${TBB_ROOT}/cmake/TBBBuild.cmake) + + # Build TBB with enabled Community Preview Features (CPF). + tbb_build(TBB_ROOT ${TBB_ROOT} CONFIG_DIR TBB_DIR MAKE_ARGS tbb_cpf=1) + + find_package(TBB REQUIRED tbb_preview) + + # Link TBB imported targets to the executable; + # "TBB::tbb_preview" can be used instead of "${TBB_IMPORTED_TARGETS}". + target_link_libraries(sub_string_finder ${TBB_IMPORTED_TARGETS}) +#. Create a build directory for the sub_string_finder sample to perform build out of source, go to the created directory + ``mkdir ~/demo_tbb_cmake/tbb/examples/GettingStarted/sub_string_finder/build ; cd ~/demo_tbb_cmake/tbb/examples/GettingStarted/sub_string_finder/build`` +#. Run CMake to prepare Makefile for the sub_string_finder sample and provide TBB location (root) where to perform build: + ``cmake -DTBB_ROOT=${HOME}/demo_tbb_cmake/tbb ..`` +#. Make an executable and run it: + ``make ; ./sub_string_finder`` + +Technical documentation for CMake modules +----------------------------------------- +TBBConfig +^^^^^^^^^ + +Configuration module for TBB library. + +How to use this module in your CMake project: + #. Add location of TBB (root) to `CMAKE_PREFIX_PATH `_ + or specify location of TBBConfig.cmake in ``TBB_DIR``. + #. Use `find_package `_ to configure TBB. + #. Use provided variables and/or imported targets (described below) to work with TBB. + +TBB components can be passed to `find_package `_ +after keyword ``COMPONENTS`` or ``REQUIRED``. +Use basic names of components (``tbb``, ``tbbmalloc``, ``tbb_preview``, etc.). + +If components are not specified then default are used: ``tbb``, ``tbbmalloc`` and ``tbbmalloc_proxy``. + +If ``tbbmalloc_proxy`` is requested, ``tbbmalloc`` component will also be added and set as dependency for ``tbbmalloc_proxy``. + +TBBConfig creates `imported targets `_ as +shared libraries using the following format: ``TBB::`` (for example, ``TBB::tbb``, ``TBB::tbbmalloc``). + +Variables set during TBB configuration: + +========================= ================================================ + Variable Description +========================= ================================================ +``TBB_FOUND`` TBB library is found +``TBB__FOUND`` specific TBB component is found +``TBB_IMPORTED_TARGETS`` all created TBB imported targets +``TBB_VERSION`` TBB version (format: ``.``) +``TBB_INTERFACE_VERSION`` TBB interface version +========================= ================================================ + +TBBInstallConfig +^^^^^^^^^^^^^^^^ + +Module for generation and installation of TBB CMake configuration files (TBBConfig.cmake and TBBConfigVersion.cmake files) on Linux, macOS and Windows. + +Provides the following functions: + + .. code:: cmake + + tbb_install_config(INSTALL_DIR SYSTEM_NAME Linux|Darwin|Windows + [TBB_VERSION ..|TBB_VERSION_FILE ] + [LIB_REL_PATH INC_REL_PATH ] + [LIB_PATH INC_PATH ])`` + +**Note: the module overwrites existing TBBConfig.cmake and TBBConfigVersion.cmake files in .** + +``tbb_config_installer.cmake`` allows to run ``TBBInstallConfig.cmake`` from command line. +It accepts the same parameters as ``tbb_install_config`` function, run ``cmake -P tbb_config_installer.cmake`` to get help. + +Use cases +""""""""" +**Prepare TBB CMake configuration files for custom TBB package.** + +The use case is applicable for package maintainers who create own TBB packages and want to create TBBConfig.cmake and TBBConfigVersion.cmake for these packages. + +=========================================== =========================================================== + Parameter Description +=========================================== =========================================================== +``INSTALL_DIR `` Directory to install CMake configuration files +``SYSTEM_NAME Linux|Darwin|Windows`` OS name to generate config files for +``TBB_VERSION_FILE `` Path to ``tbb_stddef.h`` to parse version from and + write it to TBBConfigVersion.cmake +``TBB_VERSION ..`` Directly specified TBB version; + alternative to ``TBB_VERSION_FILE`` parameter +``LIB_REL_PATH `` Relative path to TBB binaries (.lib files on Windows), default: ``../../../lib`` +``BIN_REL_PATH `` Relative path to TBB DLLs, default: ``../../../bin`` (applicable for Windows only) +``INC_REL_PATH `` Relative path to TBB headers, default: ``../../../include`` +=========================================== =========================================================== + +*Example* + + Assume your package is installed to the following structure: + + * Binaries go to ``/lib`` + * Headers go to ``/include`` + * CMake configuration files go to ``/lib/cmake/`` + + The package is packed from ``/my/package/content`` directory. + + ``cmake -DINSTALL_DIR=/my/package/content/lib/cmake/TBB -DSYSTEM_NAME=Linux -DTBB_VERSION_FILE=/my/package/content/include/tbb/tbb_stddef.h -P tbb_config_installer.cmake`` (default relative paths will be used) + +**Install TBB CMake configuration files for installed TBB.** + +The use case is applicable for users who have installed TBB, but do not have (or have incorrect) CMake configuration files for this TBB. + +==================================== ============================================== + Parameter Description +==================================== ============================================== +``INSTALL_DIR `` Directory to install CMake configuration files +``SYSTEM_NAME Linux|Darwin|Windows`` OS name to generate config files for +``LIB_PATH `` Path to installed TBB binaries (.lib files on Windows) +``BIN_PATH `` Path to installed TBB DLLs (applicable for Windows only) +``INC_PATH `` Path to installed TBB headers +==================================== ============================================== + +``LIB_PATH`` and ``INC_PATH`` will be converted to relative paths based on ``INSTALL_DIR``. +By default TBB version will be parsed from ``/tbb/tbb_stddef.h``, +but it can be overridden by optional parameters ``TBB_VERSION_FILE`` or ``TBB_VERSION``. + +*Example* + + TBB is installed to ``/usr`` directory. + In order to create TBBConfig.cmake and TBBConfigVersion.cmake in ``/usr/lib/cmake/TBB`` run + + ``cmake -DINSTALL_DIR=/usr/lib/cmake/TBB -DSYSTEM_NAME=Linux -DLIB_PATH=/usr/lib -DINC_PATH=/usr/include -P tbb_config_installer.cmake``. + +TBBGet +^^^^^^ + +Module for getting TBB library from `GitHub `_. + +Provides the following functions: + ``tbb_get(TBB_ROOT [RELEASE_TAG |LATEST] [SAVE_TO ] [SYSTEM_NAME Linux|Windows|Darwin] [CONFIG_DIR | SOURCE_CODE])`` + downloads TBB from GitHub and creates TBBConfig for the downloaded binary package if there is no TBBConfig. + + ==================================== ==================================== + Parameter Description + ==================================== ==================================== + ``TBB_ROOT `` a variable to save TBB root in, ``-NOTFOUND`` will be provided in case ``tbb_get`` is unsuccessful + ``RELEASE_TAG |LATEST`` TBB release tag to be downloaded (for example, ``2017_U6``), ``LATEST`` is used by default + ``SAVE_TO `` path to location at which to unpack downloaded TBB, ``${CMAKE_CURRENT_BINARY_DIR}/tbb_downloaded`` is used by default + ``SYSTEM_NAME Linux|Windows|Darwin`` operating system name to download a binary package for, + value of `CMAKE_SYSTEM_NAME `_ is used by default + ``CONFIG_DIR `` a variable to save location of TBBConfig.cmake and TBBConfigVersion.cmake. Ignored if ``SOURCE_CODE`` specified + ``SOURCE_CODE`` flag to get TBB source code (instead of binary package) + ==================================== ==================================== + +TBBMakeConfig +^^^^^^^^^^^^^ + +Module for making TBBConfig in `official TBB binary packages published on GitHub `_. + +This module is to be used for packages that do not have TBBConfig. + +Provides the following functions: + ``tbb_make_config(TBB_ROOT CONFIG_DIR [SYSTEM_NAME Linux|Windows|Darwin])`` + creates CMake configuration files (TBBConfig.cmake and TBBConfigVersion.cmake) for TBB binary package. + + ==================================== ==================================== + Parameter Description + ==================================== ==================================== + ``TBB_ROOT `` path to TBB root + ``CONFIG_DIR `` a variable to store location of the created configuration files + ``SYSTEM_NAME Linux|Windows|Darwin`` operating system name of the binary TBB package, + value of `CMAKE_SYSTEM_NAME `_ is used by default + ==================================== ==================================== + +TBBBuild +^^^^^^^^ + +Module for building TBB library from the source code. + +Provides the following functions: + ``tbb_build(TBB_ROOT CONFIG_DIR [MAKE_ARGS ])`` + builds TBB from source code using the ``Makefile``, creates and provides the location of the CMake configuration files (TBBConfig.cmake and TBBConfigVersion.cmake) . + + ===================================== ==================================== + Parameter Description + ===================================== ==================================== + ``TBB_ROOT `` path to TBB root + ``CONFIG_DIR `` a variable to store location of the created configuration files, + ``-NOTFOUND`` will be provided in case ``tbb_build`` is unsuccessful + ``MAKE_ARGS `` custom arguments to be passed to ``make`` tool. + + The following arguments are always passed with automatically detected values to + ``make`` tool if they are not redefined in ````: + + - ``compiler=`` + - ``tbb_build_dir=`` + - ``tbb_build_prefix=`` + - ``-j`` + ===================================== ==================================== + + +------------ + +Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries. + +``*`` Other names and brands may be claimed as the property of others. diff --git a/src/tbb/cmake/TBBBuild.cmake b/src/tbb/cmake/TBBBuild.cmake new file mode 100644 index 00000000..a2222e35 --- /dev/null +++ b/src/tbb/cmake/TBBBuild.cmake @@ -0,0 +1,197 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Usage: +# include(TBBBuild.cmake) +# tbb_build(TBB_ROOT CONFIG_DIR MAKE_ARGS [... ]) +# find_package(TBB ) +# + +include(CMakeParseArguments) + +# Save the location of Intel TBB CMake modules here, as it will not be possible to do inside functions, +# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html +set(_tbb_cmake_module_path ${CMAKE_CURRENT_LIST_DIR}) + +## +# Builds Intel TBB. +# +# Parameters: +# TBB_ROOT - path to Intel TBB root directory (with sources); +# MAKE_ARGS - user-defined arguments to be passed to make-tool; +# CONFIG_DIR - store location of the created TBBConfig if the build was ok, store -NOTFOUND otherwise. +# +function(tbb_build) + # NOTE: internal function are used to hide them from user. + + ## + # Provides arguments for make-command to build Intel TBB. + # + # Following arguments are provided automatically if they are not defined by user: + # compiler= + # tbb_build_dir= + # tbb_build_prefix= + # -j + # + # Parameters: + # USER_DEFINED_ARGS - list of user-defined arguments; + # RESULT - resulting list of 'make' arguments. + # + function(tbb_get_make_args) + set(oneValueArgs RESULT) + set(multiValueArgs USER_DEFINED_ARGS) + cmake_parse_arguments(tbb_GMA "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(result ${tbb_GMA_USER_DEFINED_ARGS}) + + if (NOT tbb_GMA_USER_DEFINED_ARGS MATCHES "compiler=") + # TODO: add other supported compilers. + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(compiler gcc) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set(compiler icc) + if (CMAKE_SYSTEM_NAME MATCHES "Windows") + set(compiler icl) + endif() + elseif (MSVC) + set(compiler cl) + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(compiler clang) + endif() + + set(result "compiler=${compiler}" ${result}) + endif() + + if (NOT tbb_GMA_USER_DEFINED_ARGS MATCHES "stdver=" AND DEFINED CMAKE_CXX_STANDARD) + set(result "stdver=c++${CMAKE_CXX_STANDARD}" ${result}) + endif() + + if (NOT tbb_GMA_USER_DEFINED_ARGS MATCHES "tbb_build_dir=") + set(result "tbb_build_dir=${CMAKE_CURRENT_BINARY_DIR}/tbb_cmake_build" ${result}) + endif() + + if (NOT tbb_GMA_USER_DEFINED_ARGS MATCHES "tbb_build_prefix=") + set(result "tbb_build_prefix=tbb_cmake_build_subdir" ${result}) + endif() + + if (NOT tbb_GMA_USER_DEFINED_ARGS MATCHES "(;|^) *\\-j[0-9]* *(;|$)") + include(ProcessorCount) + ProcessorCount(num_of_cores) + if (NOT num_of_cores EQUAL 0) + set(result "-j${num_of_cores}" ${result}) + endif() + endif() + + if (CMAKE_SYSTEM_NAME MATCHES "Android") + set(result target=android ${result}) + endif() + + set(${tbb_GMA_RESULT} ${result} PARENT_SCOPE) + endfunction() + + ## + # Provides release and debug directories basing on 'make' arguments. + # + # Following 'make' arguments are parsed: tbb_build_dir, tbb_build_prefix + # + # Parameters: + # MAKE_ARGS - 'make' arguments (tbb_build_dir and tbb_build_prefix are required) + # RELEASE_DIR - store normalized (CMake) path to release directory + # DEBUG_DIR - store normalized (CMake) path to debug directory + # + function(tbb_get_build_paths_from_make_args) + set(oneValueArgs RELEASE_DIR DEBUG_DIR) + set(multiValueArgs MAKE_ARGS) + cmake_parse_arguments(tbb_GBPFMA "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + foreach(arg ${tbb_GBPFMA_MAKE_ARGS}) + if (arg MATCHES "tbb_build_dir=") + string(REPLACE "tbb_build_dir=" "" tbb_build_dir "${arg}") + elseif (arg MATCHES "tbb_build_prefix=") + string(REPLACE "tbb_build_prefix=" "" tbb_build_prefix "${arg}") + endif() + endforeach() + + set(tbb_release_dir "${tbb_build_dir}/${tbb_build_prefix}_release") + set(tbb_debug_dir "${tbb_build_dir}/${tbb_build_prefix}_debug") + + file(TO_CMAKE_PATH "${tbb_release_dir}" tbb_release_dir) + file(TO_CMAKE_PATH "${tbb_debug_dir}" tbb_debug_dir) + + set(${tbb_GBPFMA_RELEASE_DIR} ${tbb_release_dir} PARENT_SCOPE) + set(${tbb_GBPFMA_DEBUG_DIR} ${tbb_debug_dir} PARENT_SCOPE) + endfunction() + + # -------------------- # + # Function entry point # + # -------------------- # + set(oneValueArgs TBB_ROOT CONFIG_DIR) + set(multiValueArgs MAKE_ARGS) + cmake_parse_arguments(tbb_build "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (NOT EXISTS "${tbb_build_TBB_ROOT}/Makefile" OR NOT EXISTS "${tbb_build_TBB_ROOT}/src") + message(STATUS "Intel TBB can not be built: Makefile or src directory was not found in ${tbb_build_TBB_ROOT}") + set(${tbb_build_CONFIG_DIR} ${tbb_build_CONFIG_DIR}-NOTFOUND PARENT_SCOPE) + return() + endif() + + set(make_tool_name make) + if (CMAKE_SYSTEM_NAME MATCHES "Windows") + set(make_tool_name gmake) + elseif (CMAKE_SYSTEM_NAME MATCHES "Android") + set(make_tool_name ndk-build) + endif() + + find_program(TBB_MAKE_TOOL ${make_tool_name} DOC "Make-tool to build Intel TBB.") + mark_as_advanced(TBB_MAKE_TOOL) + + if (NOT TBB_MAKE_TOOL) + message(STATUS "Intel TBB can not be built: required make-tool (${make_tool_name}) was not found") + set(${tbb_build_CONFIG_DIR} ${tbb_build_CONFIG_DIR}-NOTFOUND PARENT_SCOPE) + return() + endif() + + tbb_get_make_args(USER_DEFINED_ARGS ${tbb_build_MAKE_ARGS} RESULT tbb_make_args) + + set(tbb_build_cmd ${TBB_MAKE_TOOL} ${tbb_make_args}) + + string(REPLACE ";" " " tbb_build_cmd_str "${tbb_build_cmd}") + message(STATUS "Building Intel TBB: ${tbb_build_cmd_str}") + execute_process(COMMAND ${tbb_build_cmd} + WORKING_DIRECTORY ${tbb_build_TBB_ROOT} + RESULT_VARIABLE tbb_build_result + ERROR_VARIABLE tbb_build_error_output + OUTPUT_QUIET) + + if (NOT tbb_build_result EQUAL 0) + message(STATUS "Building is unsuccessful (${tbb_build_result}): ${tbb_build_error_output}") + set(${tbb_build_CONFIG_DIR} ${tbb_build_CONFIG_DIR}-NOTFOUND PARENT_SCOPE) + return() + endif() + + tbb_get_build_paths_from_make_args(MAKE_ARGS ${tbb_make_args} + RELEASE_DIR tbb_release_dir + DEBUG_DIR tbb_debug_dir) + + include(${_tbb_cmake_module_path}/TBBMakeConfig.cmake) + tbb_make_config(TBB_ROOT ${tbb_build_TBB_ROOT} + SYSTEM_NAME ${CMAKE_SYSTEM_NAME} + CONFIG_DIR tbb_config_dir + CONFIG_FOR_SOURCE + TBB_RELEASE_DIR ${tbb_release_dir} + TBB_DEBUG_DIR ${tbb_debug_dir}) + + set(${tbb_build_CONFIG_DIR} ${tbb_config_dir} PARENT_SCOPE) +endfunction() diff --git a/src/tbb/cmake/TBBGet.cmake b/src/tbb/cmake/TBBGet.cmake new file mode 100644 index 00000000..87872931 --- /dev/null +++ b/src/tbb/cmake/TBBGet.cmake @@ -0,0 +1,294 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(CMakeParseArguments) + +# Save the location of Intel TBB CMake modules here, as it will not be possible to do inside functions, +# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html +set(_tbb_cmake_module_path ${CMAKE_CURRENT_LIST_DIR}) + +## +# Downloads file. +# +# Parameters: +# URL - URL to download data from; +# SAVE_AS - filename there to save downloaded data; +# INFO - text description of content to be downloaded; +# will be printed as message in format is "Downloading : ; +# FORCE - option to delete local file from SAVE_AS if it exists; +# +function(_tbb_download_file) + set(options FORCE) + set(oneValueArgs URL RELEASE SAVE_AS INFO) + cmake_parse_arguments(tbb_df "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (tbb_df_FORCE AND EXISTS "${tbb_df_SAVE_AS}") + file(REMOVE ${tbb_df_SAVE_AS}) + endif() + + if (NOT EXISTS "${tbb_df_SAVE_AS}") + set(_show_progress) + if (TBB_DOWNLOADING_PROGRESS) + set(_show_progress SHOW_PROGRESS) + endif() + + message(STATUS "Downloading ${tbb_df_INFO}: ${tbb_df_URL}") + file(DOWNLOAD ${tbb_df_URL} ${tbb_df_SAVE_AS} ${_show_progress} STATUS download_status) + + list(GET download_status 0 download_status_num) + if (NOT download_status_num EQUAL 0) + message(STATUS "Unsuccessful downloading: ${download_status}") + file(REMOVE ${tbb_df_SAVE_AS}) + return() + endif() + else() + message(STATUS "Needed file was found locally ${tbb_df_SAVE_AS}. Remove it if you still want to download a new one") + endif() +endfunction() + +## +# Checks if specified Intel TBB release is available on GitHub. +# +# tbb_check_git_release( ) +# Parameters: +# - release to be checked; +# - store result (TRUE/FALSE). +# +function(_tbb_check_git_release_tag _tbb_release_tag _tbb_release_tag_avail) + if (_tbb_release_tag STREQUAL LATEST) + set(${_tbb_release_tag_avail} TRUE PARENT_SCOPE) + return() + endif() + + set(tbb_releases_file "${CMAKE_CURRENT_BINARY_DIR}/tbb_releases.json") + + _tbb_download_file(URL "${tbb_github_api}/releases" + SAVE_AS ${tbb_releases_file} + INFO "information from GitHub about Intel TBB releases" + FORCE) + + if (NOT EXISTS "${tbb_releases_file}") + set(${_tbb_release_tag_avail} FALSE PARENT_SCOPE) + return() + endif() + + file(READ ${tbb_releases_file} tbb_releases) + + string(REPLACE "\"" "" tbb_releases ${tbb_releases}) + string(REGEX MATCHALL "tag_name: *([A-Za-z0-9_\\.]+)" tbb_releases ${tbb_releases}) + + set(_release_available FALSE) + foreach(tbb_rel ${tbb_releases}) + string(REGEX REPLACE "tag_name: *" "" tbb_rel_cut ${tbb_rel}) + list(REMOVE_ITEM tbb_releases ${tbb_rel}) + list(APPEND tbb_releases ${tbb_rel_cut}) + if (_tbb_release_tag STREQUAL tbb_rel_cut) + set(_release_available TRUE) + break() + endif() + endforeach() + + if (NOT _release_available) + string(REPLACE ";" ", " tbb_releases_str "${tbb_releases}") + message(STATUS "Requested release tag ${_tbb_release_tag} is not available. Available Intel TBB release tags: ${tbb_releases_str}") + endif() + + set(${_tbb_release_tag_avail} ${_release_available} PARENT_SCOPE) +endfunction() + +## +# Compares two Intel TBB releases and provides result +# TRUE if the first release is less than the second, FALSE otherwise. +# +# tbb_is_release_less( ) +# +function(_tbb_is_release_less rel1 rel2 result) + # Convert release to numeric representation to compare it using "if" with VERSION_LESS. + string(REGEX REPLACE "[A-Za-z]" "" rel1 "${rel1}") + string(REPLACE "_" "." rel1 "${rel1}") + string(REGEX REPLACE "[A-Za-z]" "" rel2 "${rel2}") + string(REPLACE "_" "." rel2 "${rel2}") + + if (${rel1} VERSION_LESS ${rel2}) + set(${result} TRUE PARENT_SCOPE) + return() + endif() + + set(${result} FALSE PARENT_SCOPE) +endfunction() + +## +# Finds exact URL to download Intel TBB basing on provided parameters. +# +# Usage: +# _tbb_get_url(URL RELEASE_TAG OS [SOURCE_CODE]) +# +function(_tbb_get_url) + set(oneValueArgs URL RELEASE_TAG OS) + set(options SOURCE_CODE) + cmake_parse_arguments(tbb_get_url "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(tbb_github_api "https://api.github.com/repos/01org/tbb") + + _tbb_check_git_release_tag(${tbb_get_url_RELEASE_TAG} tbb_release_available) + if (NOT tbb_release_available) + set(${tbb_download_FULL_PATH} ${tbb_download_FULL_PATH}-NOTFOUND PARENT_SCOPE) + return() + endif() + + if (tbb_get_url_RELEASE_TAG STREQUAL LATEST) + set(tbb_rel_info_api_url "${tbb_github_api}/releases/latest") + else() + set(tbb_rel_info_api_url "${tbb_github_api}/releases/tags/${tbb_get_url_RELEASE_TAG}") + endif() + + set(tbb_release_info_file "${CMAKE_CURRENT_BINARY_DIR}/tbb_${tbb_get_url_RELEASE_TAG}_info.json") + + _tbb_download_file(URL ${tbb_rel_info_api_url} + SAVE_AS ${tbb_release_info_file} + INFO "information from GitHub about packages for Intel TBB ${tbb_get_url_RELEASE_TAG}" + FORCE) + + if (NOT EXISTS "${tbb_release_info_file}") + set(${tbb_get_url_URL} ${tbb_get_url_URL}-NOTFOUND PARENT_SCOPE) + return() + endif() + + file(STRINGS ${tbb_release_info_file} tbb_release_info) + + if (tbb_get_url_SOURCE_CODE) + # Find name of the latest release to get link to source archive. + if (tbb_get_url_RELEASE_TAG STREQUAL LATEST) + string(REPLACE "\"" "" tbb_release_info ${tbb_release_info}) + string(REGEX REPLACE ".*tag_name: *([A-Za-z0-9_\\.]+).*" "\\1" tbb_get_url_RELEASE_TAG "${tbb_release_info}") + endif() + + set(${tbb_get_url_URL} "https://github.com/01org/tbb/archive/${tbb_get_url_RELEASE_TAG}.tar.gz" PARENT_SCOPE) + else() + if (tbb_get_url_OS MATCHES "Linux") + set(tbb_lib_archive_suffix lin.tgz) + elseif (tbb_get_url_OS MATCHES "Windows") + set(tbb_lib_archive_suffix win.zip) + elseif (tbb_get_url_OS MATCHES "Darwin") + set(tbb_lib_archive_suffix mac.tgz) + + # Since 2017_U4 release archive for Apple has suffix "mac.tgz" instead of "osx.tgz". + if (NOT tbb_get_url_RELEASE_TAG STREQUAL "LATEST") + _tbb_is_release_less(${tbb_get_url_RELEASE_TAG} 2017_U4 release_less) + if (release_less) + set(tbb_lib_archive_suffix osx.tgz) + endif() + endif() + elseif (tbb_get_url_OS MATCHES "Android") + set(tbb_lib_archive_suffix and.tgz) + else() + message(STATUS "Currently prebuilt Intel TBB is not available for your OS (${tbb_get_url_OS})") + set(${tbb_get_url_URL} ${tbb_get_url_URL}-NOTFOUND PARENT_SCOPE) + return() + endif() + + string(REGEX REPLACE ".*(https.*oss_${tbb_lib_archive_suffix}).*" "\\1" tbb_bin_url "${tbb_release_info}") + + set(${tbb_get_url_URL} ${tbb_bin_url} PARENT_SCOPE) + endif() +endfunction() + +function(tbb_get) + set(oneValueArgs RELEASE_TAG SYSTEM_NAME SAVE_TO TBB_ROOT CONFIG_DIR) + set(options SOURCE_CODE) + cmake_parse_arguments(tbb_get "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(tbb_os ${CMAKE_SYSTEM_NAME}) + if (tbb_get_SYSTEM_NAME) + set(tbb_os ${tbb_get_SYSTEM_NAME}) + endif() + + set(tbb_release_tag LATEST) + if (tbb_get_RELEASE_TAG) + set(tbb_release_tag ${tbb_get_RELEASE_TAG}) + endif() + + set(tbb_save_to ${CMAKE_CURRENT_BINARY_DIR}/tbb_downloaded) + if (tbb_get_SAVE_TO) + set(tbb_save_to ${tbb_get_SAVE_TO}) + endif() + + if (tbb_get_SOURCE_CODE) + _tbb_get_url(URL tbb_url RELEASE_TAG ${tbb_release_tag} OS ${tbb_os} SOURCE_CODE) + else() + _tbb_get_url(URL tbb_url RELEASE_TAG ${tbb_release_tag} OS ${tbb_os}) + endif() + + if (NOT tbb_url) + message(STATUS "URL to download Intel TBB has not been found") + set(${tbb_get_TBB_ROOT} ${tbb_get_TBB_ROOT}-NOTFOUND PARENT_SCOPE) + return() + endif() + + get_filename_component(filename ${tbb_url} NAME) + set(local_file "${CMAKE_CURRENT_BINARY_DIR}/${filename}") + + _tbb_download_file(URL ${tbb_url} + SAVE_AS ${local_file} + INFO "Intel TBB library") + + if (NOT EXISTS "${local_file}") + set(${tbb_get_TBB_ROOT} ${tbb_get_TBB_ROOT}-NOTFOUND PARENT_SCOPE) + return() + endif() + + get_filename_component(subdir_name ${filename} NAME_WE) + file(MAKE_DIRECTORY ${tbb_save_to}/${subdir_name}) + if (NOT EXISTS "${tbb_save_to}/${subdir_name}") + message(STATUS "${tbb_save_to}/${subdir_name} can not be created") + set(${tbb_get_TBB_ROOT} ${tbb_get_TBB_ROOT}-NOTFOUND PARENT_SCOPE) + return() + endif() + + message(STATUS "Unpacking ${local_file} to ${tbb_save_to}/${subdir_name}") + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${local_file} + WORKING_DIRECTORY ${tbb_save_to}/${subdir_name} + RESULT_VARIABLE unpacking_result) + + if (NOT unpacking_result EQUAL 0) + message(STATUS "Unsuccessful unpacking: ${unpacking_result}") + set(${tbb_get_TBB_ROOT} ${tbb_get_TBB_ROOT}-NOTFOUND PARENT_SCOPE) + return() + endif() + + file(GLOB_RECURSE tbb_h ${tbb_save_to}/${subdir_name}/*/include/tbb/tbb.h) + list(GET tbb_h 0 tbb_h) + + if (NOT EXISTS "${tbb_h}") + message(STATUS "tbb/tbb.h has not been found in the downloaded package") + set(${tbb_get_TBB_ROOT} ${tbb_get_TBB_ROOT}-NOTFOUND PARENT_SCOPE) + return() + endif() + + get_filename_component(tbb_root "${tbb_h}" PATH) + get_filename_component(tbb_root "${tbb_root}" PATH) + get_filename_component(tbb_root "${tbb_root}" PATH) + + if (NOT tbb_get_SOURCE_CODE) + set(tbb_config_dir ${tbb_root}/cmake) + + if (NOT EXISTS "${tbb_config_dir}") + tbb_make_config(TBB_ROOT ${tbb_root} CONFIG_DIR tbb_config_dir) + endif() + + set(${tbb_get_CONFIG_DIR} ${tbb_config_dir} PARENT_SCOPE) + endif() + + set(${tbb_get_TBB_ROOT} ${tbb_root} PARENT_SCOPE) +endfunction() diff --git a/src/tbb/cmake/TBBInstallConfig.cmake b/src/tbb/cmake/TBBInstallConfig.cmake new file mode 100644 index 00000000..b6ed34b0 --- /dev/null +++ b/src/tbb/cmake/TBBInstallConfig.cmake @@ -0,0 +1,124 @@ +# Copyright (c) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(CMakeParseArguments) + +# Save the location of Intel TBB CMake modules here, as it will not be possible to do inside functions, +# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html +set(_tbb_cmake_module_path ${CMAKE_CURRENT_LIST_DIR}) + +function(tbb_install_config) + set(oneValueArgs INSTALL_DIR + SYSTEM_NAME + LIB_REL_PATH INC_REL_PATH BIN_REL_PATH TBB_VERSION TBB_VERSION_FILE + LIB_PATH BIN_PATH INC_PATH) # If TBB is installed on the system + + cmake_parse_arguments(tbb_IC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + get_filename_component(config_install_dir ${tbb_IC_INSTALL_DIR} ABSOLUTE) + file(MAKE_DIRECTORY ${config_install_dir}) + + # --- TBB_LIB_REL_PATH handling --- + set(TBB_LIB_REL_PATH "../../../lib") + + if (tbb_IC_LIB_REL_PATH) + file(TO_CMAKE_PATH ${tbb_IC_LIB_REL_PATH} TBB_LIB_REL_PATH) + endif() + + if (tbb_IC_LIB_PATH) + get_filename_component(lib_abs_path ${tbb_IC_LIB_PATH} ABSOLUTE) + file(RELATIVE_PATH TBB_LIB_REL_PATH ${config_install_dir} ${lib_abs_path}) + unset(lib_abs_path) + endif() + # ------ + + # --- TBB_BIN_REL_PATH handling --- + set(TBB_BIN_REL_PATH "../../../bin") + + if (tbb_IC_BIN_REL_PATH) + file(TO_CMAKE_PATH ${tbb_IC_BIN_REL_PATH} TBB_BIN_REL_PATH) + endif() + + if (tbb_IC_BIN_PATH) + get_filename_component(bin_abs_path ${tbb_IC_BIN_PATH} ABSOLUTE) + file(RELATIVE_PATH TBB_BIN_REL_PATH ${config_install_dir} ${bin_abs_path}) + unset(bin_abs_path) + endif() + # ------ + + # --- TBB_INC_REL_PATH handling --- + set(TBB_INC_REL_PATH "../../../include") + + if (tbb_IC_INC_REL_PATH) + file(TO_CMAKE_PATH ${tbb_IC_INC_REL_PATH} TBB_INC_REL_PATH) + endif() + + if (tbb_IC_INC_PATH) + get_filename_component(inc_abs_path ${tbb_IC_INC_PATH} ABSOLUTE) + file(RELATIVE_PATH TBB_INC_REL_PATH ${config_install_dir} ${inc_abs_path}) + unset(inc_abs_path) + endif() + # ------ + + # --- TBB_VERSION handling --- + if (tbb_IC_TBB_VERSION) + set(TBB_VERSION ${tbb_IC_TBB_VERSION}) + else() + set(tbb_version_file "${config_install_dir}/${TBB_INC_REL_PATH}/tbb/tbb_stddef.h") + if (tbb_IC_TBB_VERSION_FILE) + set(tbb_version_file ${tbb_IC_TBB_VERSION_FILE}) + endif() + + file(READ ${tbb_version_file} _tbb_stddef) + string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" _tbb_ver_major "${_tbb_stddef}") + string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" _tbb_ver_minor "${_tbb_stddef}") + string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" _tbb_ver_interface "${_tbb_stddef}") + set(TBB_VERSION "${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_interface}") + endif() + # ------ + + set(tbb_system_name ${CMAKE_SYSTEM_NAME}) + if (tbb_IC_SYSTEM_NAME) + set(tbb_system_name ${tbb_IC_SYSTEM_NAME}) + endif() + + if (tbb_system_name STREQUAL "Linux") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "so.2") + set(TBB_IMPLIB_RELEASE "") + set(TBB_IMPLIB_DEBUG "") + elseif (tbb_system_name STREQUAL "Darwin") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "dylib") + set(TBB_IMPLIB_RELEASE "") + set(TBB_IMPLIB_DEBUG "") + elseif (tbb_system_name STREQUAL "Windows") + set(TBB_LIB_PREFIX "") + set(TBB_LIB_EXT "dll") + # .lib files installed to TBB_LIB_REL_PATH (e.g. /lib); + # .dll files installed to TBB_BIN_REL_PATH (e.g. /bin); + # Expand TBB_LIB_REL_PATH here in IMPORTED_IMPLIB property and + # redefine it with TBB_BIN_REL_PATH value to properly fill IMPORTED_LOCATION property in TBBConfig.cmake.in template. + set(TBB_IMPLIB_RELEASE " + IMPORTED_IMPLIB_RELEASE \"\${CMAKE_CURRENT_LIST_DIR}/${TBB_LIB_REL_PATH}/\${_tbb_component}.lib\"") + set(TBB_IMPLIB_DEBUG " + IMPORTED_IMPLIB_DEBUG \"\${CMAKE_CURRENT_LIST_DIR}/${TBB_LIB_REL_PATH}/\${_tbb_component}_debug.lib\"") + set(TBB_LIB_REL_PATH ${TBB_BIN_REL_PATH}) + else() + message(FATAL_ERROR "Unsupported OS name: ${tbb_system_name}") + endif() + + configure_file(${_tbb_cmake_module_path}/templates/TBBConfig.cmake.in ${config_install_dir}/TBBConfig.cmake @ONLY) + configure_file(${_tbb_cmake_module_path}/templates/TBBConfigVersion.cmake.in ${config_install_dir}/TBBConfigVersion.cmake @ONLY) +endfunction() diff --git a/src/tbb/cmake/TBBMakeConfig.cmake b/src/tbb/cmake/TBBMakeConfig.cmake new file mode 100644 index 00000000..bbcb990b --- /dev/null +++ b/src/tbb/cmake/TBBMakeConfig.cmake @@ -0,0 +1,190 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Usage: +# include(TBBMakeConfig.cmake) +# tbb_make_config(TBB_ROOT SYSTEM_NAME CONFIG_DIR [SAVE_TO] [CONFIG_FOR_SOURCE TBB_RELEASE_DIR TBB_DEBUG_DIR ]) +# + +include(CMakeParseArguments) + +# Save the location of Intel TBB CMake modules here, as it will not be possible to do inside functions, +# see for details: https://cmake.org/cmake/help/latest/variable/CMAKE_CURRENT_LIST_DIR.html +set(_tbb_cmake_module_path ${CMAKE_CURRENT_LIST_DIR}) + +function(tbb_make_config) + set(oneValueArgs TBB_ROOT SYSTEM_NAME CONFIG_DIR SAVE_TO TBB_RELEASE_DIR TBB_DEBUG_DIR) + set(options CONFIG_FOR_SOURCE) + cmake_parse_arguments(tbb_MK "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(tbb_system_name ${CMAKE_SYSTEM_NAME}) + if (tbb_MK_SYSTEM_NAME) + set(tbb_system_name ${tbb_MK_SYSTEM_NAME}) + endif() + + set(tbb_config_dir ${tbb_MK_TBB_ROOT}/cmake) + if (tbb_MK_SAVE_TO) + set(tbb_config_dir ${tbb_MK_SAVE_TO}) + endif() + + file(MAKE_DIRECTORY ${tbb_config_dir}) + + set(TBB_DEFAULT_COMPONENTS tbb tbbmalloc tbbmalloc_proxy) + + if (tbb_MK_CONFIG_FOR_SOURCE) + set(TBB_RELEASE_DIR ${tbb_MK_TBB_RELEASE_DIR}) + set(TBB_DEBUG_DIR ${tbb_MK_TBB_DEBUG_DIR}) + endif() + + if (tbb_system_name STREQUAL "Linux") + set(TBB_SHARED_LIB_DIR "lib") + set(TBB_X32_SUBDIR "ia32") + set(TBB_X64_SUBDIR "intel64") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "so.2") + + # Note: multiline variable + set(TBB_CHOOSE_COMPILER_SUBDIR "if (CMAKE_CXX_COMPILER_LOADED) + set(_tbb_compiler_id \${CMAKE_CXX_COMPILER_ID}) + set(_tbb_compiler_ver \${CMAKE_CXX_COMPILER_VERSION}) +elseif (CMAKE_C_COMPILER_LOADED) + set(_tbb_compiler_id \${CMAKE_C_COMPILER_ID}) + set(_tbb_compiler_ver \${CMAKE_C_COMPILER_VERSION}) +endif() + +# For non-GCC compilers try to find version of system GCC to choose right compiler subdirectory. +if (NOT _tbb_compiler_id STREQUAL \"GNU\") + execute_process(COMMAND gcc --version OUTPUT_VARIABLE _tbb_gcc_ver_output ERROR_QUIET) + string(REGEX REPLACE \".*gcc.*([0-9]+\\\\.[0-9]+)\\\\.[0-9]+.*\" \"\\\\1\" _tbb_compiler_ver \"\${_tbb_gcc_ver_output}\") + if (NOT _tbb_compiler_ver) + message(FATAL_ERROR \"This Intel TBB package is intended to be used only in environment with available 'gcc'\") + endif() + unset(_tbb_gcc_ver_output) +endif() + +set(_tbb_compiler_subdir gcc4.1) +foreach (_tbb_gcc_version 4.1 4.4 4.7) + if (NOT _tbb_compiler_ver VERSION_LESS \${_tbb_gcc_version}) + set(_tbb_compiler_subdir gcc\${_tbb_gcc_version}) + endif() +endforeach() + +unset(_tbb_compiler_id) +unset(_tbb_compiler_ver)") + + elseif (tbb_system_name STREQUAL "Windows") + set(TBB_SHARED_LIB_DIR "bin") + set(TBB_X32_SUBDIR "ia32") + set(TBB_X64_SUBDIR "intel64") + set(TBB_LIB_PREFIX "") + set(TBB_LIB_EXT "dll") + + # Note: multiline variable + set(TBB_CHOOSE_COMPILER_SUBDIR "if (NOT MSVC) + message(FATAL_ERROR \"This Intel TBB package is intended to be used only in the project with MSVC\") +endif() + +# Detect the most relevant MSVC subdirectory +set(_tbb_msvc_1700_subdir vc11) +set(_tbb_msvc_1800_subdir vc12) +set(_tbb_msvc_1900_subdir vc14) +set(_tbb_msvc_ver \${MSVC_VERSION}) +if (MSVC_VERSION VERSION_LESS 1700) + message(FATAL_ERROR \"This Intel TBB package is intended to be used only in the project with MSVC version 1700 (vc11) or higher\") +elseif (MSVC_VERSION VERSION_GREATER 1900) + set(_tbb_msvc_ver 1900) +endif() +set(_tbb_compiler_subdir \${_tbb_msvc_\${_tbb_msvc_ver}_subdir}) +unset(_tbb_msvc_1700_subdir) +unset(_tbb_msvc_1800_subdir) +unset(_tbb_msvc_1900_subdir) + +if (WINDOWS_STORE) + set(_tbb_compiler_subdir \${_tbb_compiler_subdir}_ui) +endif()") + + if (tbb_MK_CONFIG_FOR_SOURCE) + set(TBB_IMPLIB_RELEASE " + IMPORTED_IMPLIB_RELEASE \"${tbb_MK_TBB_RELEASE_DIR}/\${_tbb_component}.lib\"") + set(TBB_IMPLIB_DEBUG " + IMPORTED_IMPLIB_DEBUG \"${tbb_MK_TBB_DEBUG_DIR}/\${_tbb_component}_debug.lib\"") + else() + set(TBB_IMPLIB_RELEASE " + IMPORTED_IMPLIB_RELEASE \"\${_tbb_root}/lib/\${_tbb_arch_subdir}/\${_tbb_compiler_subdir}/\${_tbb_component}.lib\"") + set(TBB_IMPLIB_DEBUG " + IMPORTED_IMPLIB_DEBUG \"\${_tbb_root}/lib/\${_tbb_arch_subdir}/\${_tbb_compiler_subdir}/\${_tbb_component}_debug.lib\"") + endif() + + # Note: multiline variable + # tbb/internal/_tbb_windef.h (included via tbb/tbb_stddef.h) does implicit linkage of some .lib files, use a special define to avoid it + set(TBB_COMPILE_DEFINITIONS " + INTERFACE_COMPILE_DEFINITIONS \"__TBB_NO_IMPLICIT_LINKAGE=1\"") + elseif (tbb_system_name STREQUAL "Darwin") + set(TBB_SHARED_LIB_DIR "lib") + set(TBB_X32_SUBDIR ".") + set(TBB_X64_SUBDIR ".") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "dylib") + set(TBB_CHOOSE_COMPILER_SUBDIR "set(_tbb_compiler_subdir .)") + elseif (tbb_system_name STREQUAL "Android") + set(TBB_SHARED_LIB_DIR "lib") + set(TBB_X32_SUBDIR ".") + set(TBB_X64_SUBDIR "x86_64") + set(TBB_LIB_PREFIX "lib") + set(TBB_LIB_EXT "so") + set(TBB_CHOOSE_COMPILER_SUBDIR "set(_tbb_compiler_subdir .)") + else() + message(FATAL_ERROR "Unsupported OS name: ${tbb_system_name}") + endif() + + file(READ "${tbb_MK_TBB_ROOT}/include/tbb/tbb_stddef.h" _tbb_stddef) + string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" _tbb_ver_major "${_tbb_stddef}") + string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" _tbb_ver_minor "${_tbb_stddef}") + string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" TBB_INTERFACE_VERSION "${_tbb_stddef}") + set(TBB_VERSION "${_tbb_ver_major}.${_tbb_ver_minor}.${TBB_INTERFACE_VERSION}") + + if (tbb_MK_CONFIG_FOR_SOURCE) + set(TBB_CHOOSE_ARCH_AND_COMPILER "") + set(TBB_RELEASE_LIB_PATH "${TBB_RELEASE_DIR}") + set(TBB_DEBUG_LIB_PATH "${TBB_DEBUG_DIR}") + set(TBB_UNSET_ADDITIONAL_VARIABLES "") + else() + # Note: multiline variable + set(TBB_CHOOSE_ARCH_AND_COMPILER " +if (CMAKE_SIZEOF_VOID_P EQUAL 8) + set(_tbb_arch_subdir ${TBB_X64_SUBDIR}) +else() + set(_tbb_arch_subdir ${TBB_X32_SUBDIR}) +endif() + +${TBB_CHOOSE_COMPILER_SUBDIR} + +get_filename_component(_tbb_lib_path \"\${_tbb_root}/${TBB_SHARED_LIB_DIR}/\${_tbb_arch_subdir}/\${_tbb_compiler_subdir}\" ABSOLUTE) +") + + set(TBB_RELEASE_LIB_PATH "\${_tbb_lib_path}") + set(TBB_DEBUG_LIB_PATH "\${_tbb_lib_path}") + + # Note: multiline variable + set(TBB_UNSET_ADDITIONAL_VARIABLES " +unset(_tbb_arch_subdir) +unset(_tbb_compiler_subdir)") + endif() + + configure_file(${_tbb_cmake_module_path}/templates/TBBConfigInternal.cmake.in ${tbb_config_dir}/TBBConfig.cmake @ONLY) + configure_file(${_tbb_cmake_module_path}/templates/TBBConfigVersion.cmake.in ${tbb_config_dir}/TBBConfigVersion.cmake @ONLY) + + set(${tbb_MK_CONFIG_DIR} ${tbb_config_dir} PARENT_SCOPE) +endfunction() diff --git a/src/tbb/cmake/tbb_config_generator.cmake b/src/tbb/cmake/tbb_config_generator.cmake new file mode 100644 index 00000000..3f94efd1 --- /dev/null +++ b/src/tbb/cmake/tbb_config_generator.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function(tbb_conf_gen_print_help) + message("Usage: cmake -DTBB_ROOT= -DTBB_OS=Linux|Windows|Darwin [-DSAVE_TO=] -P tbb_config_generator.cmake") +endfunction() + +if (NOT DEFINED TBB_ROOT) + tbb_conf_gen_print_help() + message(FATAL_ERROR "Required parameter TBB_ROOT is not defined") +endif() + +if (NOT EXISTS "${TBB_ROOT}") + tbb_conf_gen_print_help() + message(FATAL_ERROR "TBB_ROOT=${TBB_ROOT} does not exist") +endif() + +if (NOT DEFINED TBB_OS) + tbb_conf_gen_print_help() + message(FATAL_ERROR "Required parameter TBB_OS is not defined") +endif() + +if (DEFINED SAVE_TO) + set(tbb_conf_gen_save_to_param SAVE_TO ${SAVE_TO}) +endif() + +include(${CMAKE_CURRENT_LIST_DIR}/TBBMakeConfig.cmake) +tbb_make_config(TBB_ROOT ${TBB_ROOT} CONFIG_DIR tbb_config_dir SYSTEM_NAME ${TBB_OS} ${tbb_conf_gen_save_to_param}) + +message(STATUS "TBBConfig files were created in ${tbb_config_dir}") diff --git a/src/tbb/cmake/tbb_config_installer.cmake b/src/tbb/cmake/tbb_config_installer.cmake new file mode 100644 index 00000000..fa165e8e --- /dev/null +++ b/src/tbb/cmake/tbb_config_installer.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function(tbb_conf_gen_print_help) + message("Usage: cmake -DINSTALL_DIR= -DSYSTEM_NAME=Linux|Darwin|Windows -P tbb_config_generator.cmake + +Parameters: + For custom TBB package: + -DTBB_VERSION_FILE= + -DTBB_VERSION=.. (alternative to TBB_VERSION_FILE) + -DINC_REL_PATH= + -DLIB_REL_PATH= + -DBIN_REL_PATH= (only for Windows) + For installed TBB: + -DINC_PATH= + -DLIB_PATH= + -DBIN_PATH= (only for Windows) +") +endfunction() + +if (NOT DEFINED INSTALL_DIR) + tbb_conf_gen_print_help() + message(FATAL_ERROR "Required parameter INSTALL_DIR is not defined") +endif() + +if (NOT DEFINED SYSTEM_NAME) + tbb_conf_gen_print_help() + message(FATAL_ERROR "Required parameter SYSTEM_NAME is not defined") +endif() + +foreach (arg TBB_VERSION INC_REL_PATH LIB_REL_PATH BIN_REL_PATH TBB_VERSION_FILE INC_PATH LIB_PATH BIN_PATH) + set(optional_args ${optional_args} ${arg} ${${arg}}) +endforeach() + +include(${CMAKE_CURRENT_LIST_DIR}/TBBInstallConfig.cmake) +tbb_install_config(INSTALL_DIR ${INSTALL_DIR} SYSTEM_NAME ${SYSTEM_NAME} ${optional_args}) +message(STATUS "TBBConfig files were created in ${INSTALL_DIR}") diff --git a/src/tbb/cmake/templates/TBBConfig.cmake.in b/src/tbb/cmake/templates/TBBConfig.cmake.in new file mode 100644 index 00000000..84e25399 --- /dev/null +++ b/src/tbb/cmake/templates/TBBConfig.cmake.in @@ -0,0 +1,95 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It defines the following variables: +# TBB_tbb_FOUND +# TBB_tbbmalloc_FOUND +# TBB_tbbmalloc_proxy_FOUND +# TBB_IMPORTED_TARGETS +# +# TBBConfigVersion.cmake defines TBB_VERSION +# +# Initialize to default values +if (NOT TBB_tbb_FOUND) + set(TBB_tbb_FOUND 0) +endif() +if (NOT TBB_tbbmalloc_FOUND) + set(TBB_tbbmalloc_FOUND 0) +endif() +if (NOT TBB_tbbmalloc_proxy_FOUND) + set(TBB_tbbmalloc_proxy_FOUND 0) +endif() +if (NOT TBB_IMPORTED_TARGETS) + set(TBB_IMPORTED_TARGETS "") +endif() + +if (NOT TBB_FIND_COMPONENTS) + set(TBB_FIND_COMPONENTS "tbb;tbbmalloc;tbbmalloc_proxy") + foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + set(TBB_FIND_REQUIRED_${_tbb_component} 1) + endforeach() +endif() + +# Add components with internal dependencies: tbbmalloc_proxy -> tbbmalloc +list(FIND TBB_FIND_COMPONENTS tbbmalloc_proxy _tbbmalloc_proxy_ix) +if (NOT _tbbmalloc_proxy_ix EQUAL -1) + list(FIND TBB_FIND_COMPONENTS tbbmalloc _tbbmalloc_ix) + if (_tbbmalloc_ix EQUAL -1) + list(APPEND TBB_FIND_COMPONENTS tbbmalloc) + set(TBB_FIND_REQUIRED_tbbmalloc ${TBB_FIND_REQUIRED_tbbmalloc_proxy}) + endif() + unset(_tbbmalloc_ix) +endif() +unset(_tbbmalloc_proxy_ix) + +foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + set(_tbb_release_lib "${CMAKE_CURRENT_LIST_DIR}/@TBB_LIB_REL_PATH@/@TBB_LIB_PREFIX@${_tbb_component}.@TBB_LIB_EXT@") + set(_tbb_debug_lib "${CMAKE_CURRENT_LIST_DIR}/@TBB_LIB_REL_PATH@/@TBB_LIB_PREFIX@${_tbb_component}_debug.@TBB_LIB_EXT@") + + if (EXISTS "${_tbb_release_lib}" OR EXISTS "${_tbb_debug_lib}") + if (NOT TARGET TBB::${_tbb_component}) + add_library(TBB::${_tbb_component} SHARED IMPORTED) + set_target_properties(TBB::${_tbb_component} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/@TBB_INC_REL_PATH@") + + if (EXISTS "${_tbb_release_lib}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_RELEASE "${_tbb_release_lib}"@TBB_IMPLIB_RELEASE@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + endif() + + if (EXISTS "${_tbb_debug_lib}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_DEBUG "${_tbb_debug_lib}"@TBB_IMPLIB_DEBUG@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) + endif() + + # Add internal dependencies for imported targets: TBB::tbbmalloc_proxy -> TBB::tbbmalloc + if (_tbb_component STREQUAL tbbmalloc_proxy) + set_target_properties(TBB::tbbmalloc_proxy PROPERTIES INTERFACE_LINK_LIBRARIES TBB::tbbmalloc) + endif() + list(APPEND TBB_IMPORTED_TARGETS TBB::${_tbb_component}) + else() + message(STATUS "Using previously found TBB::${_tbb_component}") + endif() + set(TBB_${_tbb_component}_FOUND 1) + elseif (TBB_FIND_REQUIRED AND TBB_FIND_REQUIRED_${_tbb_component}) + message(STATUS "Missed required Intel TBB component: ${_tbb_component}") + message(STATUS " one or both of:\n ${_tbb_release_lib}\n ${_tbb_debug_lib}\n files must exist.") + set(TBB_FOUND FALSE) + set(TBB_${_tbb_component}_FOUND 0) + endif() +endforeach() +unset(_tbb_release_lib) +unset(_tbb_debug_lib) diff --git a/src/tbb/cmake/templates/TBBConfigInternal.cmake.in b/src/tbb/cmake/templates/TBBConfigInternal.cmake.in new file mode 100644 index 00000000..40528c6d --- /dev/null +++ b/src/tbb/cmake/templates/TBBConfigInternal.cmake.in @@ -0,0 +1,79 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TBB_FOUND should not be set explicitly. It is defined automatically by CMake. +# Handling of TBB_VERSION is in TBBConfigVersion.cmake. + +if (NOT TBB_FIND_COMPONENTS) + set(TBB_FIND_COMPONENTS "@TBB_DEFAULT_COMPONENTS@") + foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + set(TBB_FIND_REQUIRED_${_tbb_component} 1) + endforeach() +endif() + +# Add components with internal dependencies: tbbmalloc_proxy -> tbbmalloc +list(FIND TBB_FIND_COMPONENTS tbbmalloc_proxy _tbbmalloc_proxy_ix) +if (NOT _tbbmalloc_proxy_ix EQUAL -1) + list(FIND TBB_FIND_COMPONENTS tbbmalloc _tbbmalloc_ix) + if (_tbbmalloc_ix EQUAL -1) + list(APPEND TBB_FIND_COMPONENTS tbbmalloc) + set(TBB_FIND_REQUIRED_tbbmalloc ${TBB_FIND_REQUIRED_tbbmalloc_proxy}) + endif() +endif() + +set(TBB_INTERFACE_VERSION @TBB_INTERFACE_VERSION@) + +get_filename_component(_tbb_root "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_tbb_root "${_tbb_root}" PATH) +@TBB_CHOOSE_ARCH_AND_COMPILER@ +foreach (_tbb_component ${TBB_FIND_COMPONENTS}) + set(_tbb_release_lib "@TBB_RELEASE_LIB_PATH@/@TBB_LIB_PREFIX@${_tbb_component}.@TBB_LIB_EXT@") + set(_tbb_debug_lib "@TBB_DEBUG_LIB_PATH@/@TBB_LIB_PREFIX@${_tbb_component}_debug.@TBB_LIB_EXT@") + + if (EXISTS "${_tbb_release_lib}" OR EXISTS "${_tbb_debug_lib}") + add_library(TBB::${_tbb_component} SHARED IMPORTED) + set_target_properties(TBB::${_tbb_component} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_tbb_root}/include"@TBB_COMPILE_DEFINITIONS@) + + if (EXISTS "${_tbb_release_lib}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_RELEASE "${_tbb_release_lib}"@TBB_IMPLIB_RELEASE@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + endif() + + if (EXISTS "${_tbb_debug_lib}") + set_target_properties(TBB::${_tbb_component} PROPERTIES + IMPORTED_LOCATION_DEBUG "${_tbb_debug_lib}"@TBB_IMPLIB_DEBUG@) + set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) + endif() + + # Add internal dependencies for imported targets: TBB::tbbmalloc_proxy -> TBB::tbbmalloc + if (_tbb_component STREQUAL tbbmalloc_proxy) + set_target_properties(TBB::tbbmalloc_proxy PROPERTIES INTERFACE_LINK_LIBRARIES TBB::tbbmalloc) + endif() + + list(APPEND TBB_IMPORTED_TARGETS TBB::${_tbb_component}) + set(TBB_${_tbb_component}_FOUND 1) + elseif (TBB_FIND_REQUIRED AND TBB_FIND_REQUIRED_${_tbb_component}) + message(STATUS "Missed required Intel TBB component: ${_tbb_component}") + set(TBB_FOUND FALSE) + set(TBB_${_tbb_component}_FOUND 0) + endif() +endforeach() +@TBB_UNSET_ADDITIONAL_VARIABLES@ +unset(_tbbmalloc_proxy_ix) +unset(_tbbmalloc_ix) +unset(_tbb_lib_path) +unset(_tbb_release_lib) +unset(_tbb_debug_lib) diff --git a/src/tbb/cmake/templates/TBBConfigVersion.cmake.in b/src/tbb/cmake/templates/TBBConfigVersion.cmake.in new file mode 100644 index 00000000..2e31c80e --- /dev/null +++ b/src/tbb/cmake/templates/TBBConfigVersion.cmake.in @@ -0,0 +1,24 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(PACKAGE_VERSION @TBB_VERSION@) + +if ("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() diff --git a/src/tbb/doc/Release_Notes.txt b/src/tbb/doc/Release_Notes.txt new file mode 100644 index 00000000..0ed0c2e2 --- /dev/null +++ b/src/tbb/doc/Release_Notes.txt @@ -0,0 +1,132 @@ +------------------------------------------------------------------------ +Intel(R) Threading Building Blocks - Release Notes + Version 2019 +------------------------------------------------------------------------ + + +System Requirements +------------------- + +Intel(R) Threading Building Blocks (Intel(R) TBB) is available +commercially (see http://software.intel.com/en-us/intel-tbb) as a +binary distribution, and in open source, in both source and binary +forms (see http://threadingbuildingblocks.org). + +When built from source, Intel(R) TBB is intended to be highly portable +and so supports a wide variety of operating systems and platforms (see +http://threadingbuildingblocks.org for more details). + +Binary distributions, including commercial distributions, are validated +and officially supported for the hardware, software, operating systems +and compilers listed here. + +Hardware - Recommended + + Microsoft* Windows* Systems + Intel(R) Core(TM) processor family + Intel(R) Xeon(R) processor family + Intel(R) Xeon Phi(TM) processor family + Linux* Systems + Intel(R) Core(TM) processor family + Intel(R) Xeon(R) processor family + Intel(R) Xeon Phi(TM) processor family + macOS* Systems + Intel(R) Core(TM) processor family + Android* Systems + Intel(R) Atom(TM) processor family + +Hardware - Supported + + Intel(R) Pentium(R) 4 processor family + Intel(R) Xeon Phi(TM) coprocessor + Intel(R) Atom(TM) processor family + Non Intel(R) processors compatible with the above processors + +Software - Minimum Requirements + + Supported operating system (see below) + Supported compiler (see below) + +Software - Recommended + + Intel(R) Parallel Studio XE 2018, 2019 + Intel(R) System Studio 2018, 2019 + +Software - Supported Operating Systems + + Systems with Microsoft* Windows* operating systems + Microsoft* Windows* 10 + Microsoft* Windows* 8.1 + Microsoft* Windows* 7 SP1 + Microsoft* Windows* Server 2016 + Microsoft* Windows* Server 2012 R2 + Systems with Linux* operating systems + CentOS 7.1 + Debian* 8, 9 + Fedora* 27 + Intel(R) Cluster Ready + Red Hat* Enterprise Linux* 6, 7 + SuSE* Linux* Enterprise Server 12 + Ubuntu* 14.04 LTS, 16.04 LTS, 18.04 LTS + WindRiver* Linux 8, 9 + Yocto 2.3 + Systems with OS X* or macOS* operating systems + OS X* 10.11 + macOS* 10.12, 10.13 + Systems with Android* operating systems + Android* 5.x, 6.x, 7.x, 8.x + +Software - Supported Compilers + + Intel(R) C++ Compiler 17, 18 and 19 version + Microsoft* Visual C++ 12.0 (Microsoft* Visual Studio* 2013, + Windows* OS only) + Microsoft* Visual C++ 14.0 (Microsoft* Visual Studio* 2015, + Windows* OS only) + Microsoft* Visual C++ 14.1 (Microsoft* Visual Studio* 2017, + Windows* OS only) + Microsoft* Visual C++ 14.2 (Microsoft* Visual Studio* 2019, + Windows* OS only) + Microsoft* Windows* Software Development Kit for Windows* 8.1 + Microsoft* Windows* Software Development Kit for Windows* 10 + For each supported Linux* operating system, the standard gcc + version provided with that operating system is supported + GNU Compilers (gcc) 4.1 - 7.1 + GNU C Library (glibc) version 2.4 - 2.19 + Clang* 3.8 - 7.0 + Xcode* 7.0 - 9.1 + Android* NDK r10e - r17b + +Software - Supported Performance Analysis Tools + + Intel(R) VTune(TM) Amplifier XE 2018, 2019 + Intel(R) Inspector XE 2018, 2019 + Intel(R) Advisor XE 2018, 2019 + +Known Issues +------------ + +Below is the list of known issues in this release of +Intel(R) Threading Building Blocks (Intel(R) TBB). +See the "Known Issues" appendix in the Intel(R) TBB Developer +Reference for notes applicable to multiple releases of Intel(R) TBB. + +Library Issues + + - If you build Intel(R) TBB from sources with GCC 6, specify + the -flifetime-dse=1 option to prevent crashes at runtime, + or use Intel(R) TBB makefiles that automatically set this option. + +------------------------------------------------------------------------ +(C) 2019 Intel Corporation + +Intel, the Intel logo, Intel Core, Intel Atom, Xeon, Intel Xeon Phi and +Pentium are trademarks of Intel Corporation in the U.S. and/or other +countries. + +* Other names and brands may be claimed as the property of others. + +Third Party and Open Source Licenses + +Content of some examples or binaries may be covered by various open-source +licenses. See the index.html file in each respective folder for details. diff --git a/src/tbb/doc/copyright_brand_disclaimer_doxygen.txt b/src/tbb/doc/copyright_brand_disclaimer_doxygen.txt new file mode 100644 index 00000000..5a51272d --- /dev/null +++ b/src/tbb/doc/copyright_brand_disclaimer_doxygen.txt @@ -0,0 +1,9 @@ +
+

+Copyright © 2005-2019 Intel Corporation. All Rights Reserved. +

+Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are +registered trademarks or trademarks of Intel Corporation or its +subsidiaries in the United States and other countries. +

+* Other names and brands may be claimed as the property of others. diff --git a/src/tbb/include/index.html b/src/tbb/include/index.html index b0962e01..7895ced2 100644 --- a/src/tbb/include/index.html +++ b/src/tbb/include/index.html @@ -15,7 +15,7 @@

Directories

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/include/serial/tbb/parallel_for.h b/src/tbb/include/serial/tbb/parallel_for.h index 83ca9d57..5b42a670 100644 --- a/src/tbb/include/serial/tbb/parallel_for.h +++ b/src/tbb/include/serial/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_SERIAL_parallel_for_H diff --git a/src/tbb/include/serial/tbb/tbb_annotate.h b/src/tbb/include/serial/tbb/tbb_annotate.h index 81c8d3a4..6b79be2d 100644 --- a/src/tbb/include/serial/tbb/tbb_annotate.h +++ b/src/tbb/include/serial/tbb/tbb_annotate.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_annotate_H diff --git a/src/tbb/include/tbb/aggregator.h b/src/tbb/include/tbb/aggregator.h index 6aecbb74..33e83679 100644 --- a/src/tbb/include/tbb/aggregator.h +++ b/src/tbb/include/tbb/aggregator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__aggregator_H diff --git a/src/tbb/include/tbb/aligned_space.h b/src/tbb/include/tbb/aligned_space.h index ee4d9e72..03b09d09 100644 --- a/src/tbb/include/tbb/aligned_space.h +++ b/src/tbb/include/tbb/aligned_space.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_aligned_space_H diff --git a/src/tbb/include/tbb/atomic.h b/src/tbb/include/tbb/atomic.h index 72ec534e..92d1e11a 100644 --- a/src/tbb/include/tbb/atomic.h +++ b/src/tbb/include/tbb/atomic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_atomic_H diff --git a/src/tbb/include/tbb/blocked_range.h b/src/tbb/include/tbb/blocked_range.h index 1f3901eb..d1ff1f45 100644 --- a/src/tbb/include/tbb/blocked_range.h +++ b/src/tbb/include/tbb/blocked_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range_H @@ -25,6 +21,15 @@ namespace tbb { +namespace internal { + +// blocked_rangeNd_impl forward declaration in tbb::internal namespace to +// name it as a friend for a tbb::blocked_range. +template +class blocked_rangeNd_impl; + +} // namespace internal + /** \page range_req Requirements on range concept Class \c R implementing the concept of range must define: - \code R::R( const R& ); \endcode Copy constructor @@ -47,9 +52,11 @@ class blocked_range { //! Type for size of a range typedef std::size_t size_type; +#if __TBB_DEPRECATED_BLOCKED_RANGE_DEFAULT_CTOR //! Construct range with default-constructed values for begin, end, and grainsize. /** Requires that Value have a default constructor. */ blocked_range() : my_end(), my_begin(), my_grainsize() {} +#endif //! Construct range over half-open interval [begin,end), with the given grainsize. blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : @@ -151,6 +158,9 @@ class blocked_range { template friend class blocked_range3d; + + template + friend class internal::blocked_rangeNd_impl; }; } // namespace tbb diff --git a/src/tbb/include/tbb/blocked_range2d.h b/src/tbb/include/tbb/blocked_range2d.h index 1e3dfd17..cd0fe1c3 100644 --- a/src/tbb/include/tbb/blocked_range2d.h +++ b/src/tbb/include/tbb/blocked_range2d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range2d_H @@ -55,7 +51,7 @@ class blocked_range2d { //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_rows.empty() || my_cols.empty(); } diff --git a/src/tbb/include/tbb/blocked_range3d.h b/src/tbb/include/tbb/blocked_range3d.h index 5f3b3c86..5c6cf9f0 100644 --- a/src/tbb/include/tbb/blocked_range3d.h +++ b/src/tbb/include/tbb/blocked_range3d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_blocked_range3d_H @@ -61,7 +57,7 @@ class blocked_range3d { //! True if range is empty bool empty() const { - // Yes, it is a logical OR here, not AND. + // Range is empty if at least one dimension is empty. return my_pages.empty() || my_rows.empty() || my_cols.empty(); } diff --git a/src/tbb/include/tbb/blocked_rangeNd.h b/src/tbb/include/tbb/blocked_rangeNd.h new file mode 100644 index 00000000..b623d002 --- /dev/null +++ b/src/tbb/include/tbb/blocked_rangeNd.h @@ -0,0 +1,150 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_rangeNd_H +#define __TBB_blocked_rangeNd_H + +#if ! TBB_PREVIEW_BLOCKED_RANGE_ND + #error Set TBB_PREVIEW_BLOCKED_RANGE_ND to include blocked_rangeNd.h +#endif + +#include "tbb_config.h" + +// tbb::blocked_rangeNd requires C++11 support +#if __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +#include "internal/_template_helpers.h" // index_sequence, make_index_sequence + +#include +#include // std::any_of +#include // std::is_same, std::enable_if + +#include "tbb/blocked_range.h" + +namespace tbb { +namespace internal { + +/* + The blocked_rangeNd_impl uses make_index_sequence to automatically generate a ctor with + exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity + to use braced-init-list parameters to initialize each dimension. + Use of parameters, whose representation is a braced-init-list, but they're not + std::initializer_list or a reference to one, produces a non-deduced context + within template argument deduction. + + NOTE: blocked_rangeNd must be exactly a templated alias to the blocked_rangeNd_impl + (and not e.g. a derived class), otherwise it would need to declare its own ctor + facing the same problem that the impl class solves. +*/ + +template> +class blocked_rangeNd_impl; + +template +class blocked_rangeNd_impl> { +public: + //! Type of a value. + using value_type = Value; + +private: + + //! Helper type to construct range with N tbb::blocked_range objects. + template + using dim_type_helper = tbb::blocked_range; + +public: + blocked_rangeNd_impl() = delete; + + //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range. + blocked_rangeNd_impl(const dim_type_helper&... args) : my_dims{ {args...} } {} + + //! Dimensionality of a range. + static constexpr unsigned int ndims() { return N; } + + //! Range in certain dimension. + const tbb::blocked_range& dim(unsigned int dimension) const { + __TBB_ASSERT(dimension < N, "out of bound"); + return my_dims[dimension]; + } + + //------------------------------------------------------------------------ + // Methods that implement Range concept + //------------------------------------------------------------------------ + + //! True if at least one dimension is empty. + bool empty() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.empty(); + }); + } + + //! True if at least one dimension is divisible. + bool is_divisible() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { + return d.is_divisible(); + }); + } + +#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES + //! Static field to support proportional split. + static const bool is_splittable_in_proportion = true; + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, proportional_split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } +#endif + + blocked_rangeNd_impl(blocked_rangeNd_impl& r, split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + +private: + __TBB_STATIC_ASSERT(N != 0, "zero dimensional blocked_rangeNd can't be constructed"); + + //! Ranges in each dimension. + std::array, N> my_dims; + + template + void do_split(blocked_rangeNd_impl& r, split_type proportion) { + __TBB_STATIC_ASSERT((is_same_type::value + || is_same_type::value), + "type of split object is incorrect"); + __TBB_ASSERT(r.is_divisible(), "can't split not divisible range"); + + auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& first, const tbb::blocked_range& second) { + return (first.size() * second.grainsize() < second.size() * first.grainsize()); + }); + + auto r_it = r.my_dims.begin() + (my_it - my_dims.begin()); + + my_it->my_begin = tbb::blocked_range::do_split(*r_it, proportion); + + // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to + // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept + __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin), + "blocked_range has been split incorrectly"); + } +}; + +} // namespace internal + +template +using blocked_rangeNd = internal::blocked_rangeNd_impl; + +} // namespace tbb + +#endif /* __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ +#endif /* __TBB_blocked_rangeNd_H */ diff --git a/src/tbb/include/tbb/cache_aligned_allocator.h b/src/tbb/include/tbb/cache_aligned_allocator.h index d435e785..a9983298 100644 --- a/src/tbb/include/tbb/cache_aligned_allocator.h +++ b/src/tbb/include/tbb/cache_aligned_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_cache_aligned_allocator_H @@ -24,7 +20,11 @@ #include #include "tbb_stddef.h" #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward +#include // std::forward +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include #endif namespace tbb { @@ -69,7 +69,6 @@ class cache_aligned_allocator { template struct rebind { typedef cache_aligned_allocator other; }; - cache_aligned_allocator() throw() {} cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} template cache_aligned_allocator(const cache_aligned_allocator&) throw() {} @@ -132,6 +131,79 @@ inline bool operator==( const cache_aligned_allocator&, const cache_aligned_a template inline bool operator!=( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return false;} +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource wrapper to ensure cache line size alignment +class cache_aligned_resource : public std::pmr::memory_resource { +public: + cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} + explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} + + std::pmr::memory_resource* upstream_resource() const { + return m_upstream; + } + +private: + //! We don't know what memory resource set. Use padding to guarantee alignment + void* do_allocate(size_t bytes, size_t alignment) override { + size_t cache_line_alignment = correct_alignment(alignment); + uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment); + __TBB_ASSERT(base != 0, "Upstream resource returned NULL."); +#if _MSC_VER && !defined(__INTEL_COMPILER) + // unary minus operator applied to unsigned type, result still unsigned + #pragma warning(push) + #pragma warning(disable: 4146 4706) +#endif + // Round up to the next cache line (align the base address) + uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment; +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + // Record where block actually starts. + ((uintptr_t*)result)[-1] = base; + return (void*)result; + } + + void do_deallocate(void* ptr, size_t bytes, size_t alignment) override { + if (ptr) { + // Recover where block actually starts + uintptr_t base = ((uintptr_t*)ptr)[-1]; + m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment)); + } + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + if (this == &other) { return true; } +#if __TBB_USE_OPTIONAL_RTTI + const cache_aligned_resource* other_res = dynamic_cast(&other); + return other_res && (this->upstream_resource() == other_res->upstream_resource()); +#else + return false; +#endif + } + + size_t correct_alignment(size_t alignment) { + __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2"); +#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT + size_t cache_line_size = std::hardware_destructive_interference_size; +#else + size_t cache_line_size = internal::NFS_GetLineSize(); +#endif + return alignment < cache_line_size ? cache_line_size : alignment; + } + + size_t correct_size(size_t bytes) { + // To handle the case, when small size requested. There could be not + // enough space to store the original pointer. + return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes; + } + + std::pmr::memory_resource* m_upstream; +}; + +#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ + } // namespace tbb #endif /* __TBB_cache_aligned_allocator_H */ + diff --git a/src/tbb/include/tbb/combinable.h b/src/tbb/include/tbb/combinable.h index 0063dbb4..a8aaf61c 100644 --- a/src/tbb/include/tbb/combinable.h +++ b/src/tbb/include/tbb/combinable.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_combinable_H diff --git a/src/tbb/include/tbb/compat/condition_variable b/src/tbb/include/tbb/compat/condition_variable index 43edfc03..8dc4e913 100644 --- a/src/tbb/include/tbb/compat/condition_variable +++ b/src/tbb/include/tbb/compat/condition_variable @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_condition_variable_H diff --git a/src/tbb/include/tbb/compat/ppl.h b/src/tbb/include/tbb/compat/ppl.h index 840dfb22..a134244a 100644 --- a/src/tbb/include/tbb/compat/ppl.h +++ b/src/tbb/include/tbb/compat/ppl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_compat_ppl_H diff --git a/src/tbb/include/tbb/compat/thread b/src/tbb/include/tbb/compat/thread index 0edd9289..9dac4305 100644 --- a/src/tbb/include/tbb/compat/thread +++ b/src/tbb/include/tbb/compat/thread @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_thread_H diff --git a/src/tbb/include/tbb/compat/tuple b/src/tbb/include/tbb/compat/tuple index 5767c49e..86e6cd9f 100644 --- a/src/tbb/include/tbb/compat/tuple +++ b/src/tbb/include/tbb/compat/tuple @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tuple_H diff --git a/src/tbb/include/tbb/concurrent_hash_map.h b/src/tbb/include/tbb/concurrent_hash_map.h index b25550a4..e3f2a401 100644 --- a/src/tbb/include/tbb/concurrent_hash_map.h +++ b/src/tbb/include/tbb/concurrent_hash_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_hash_map_H @@ -27,13 +23,15 @@ #include // Need std::memset #include __TBB_STD_SWAP_HEADER -#include "cache_aligned_allocator.h" #include "tbb_allocator.h" #include "spin_rw_mutex.h" #include "atomic.h" #include "tbb_exception.h" #include "tbb_profiling.h" +#include "aligned_space.h" #include "internal/_tbb_hash_compare_impl.h" +#include "internal/_template_helpers.h" +#include "internal/_allocator_traits.h" #if __TBB_INITIALIZER_LISTS_PRESENT #include #endif @@ -43,12 +41,17 @@ #if __TBB_STATISTICS #include #endif +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT +// Definition of __TBB_CPP11_RVALUE_REF_PRESENT includes __TBB_CPP11_TUPLE_PRESENT +// for most of platforms, tuple present macro was added for logical correctness +#include +#endif namespace tbb { namespace interface5 { - template, typename A = tbb_allocator > > + template, typename A = tbb_allocator > > class concurrent_hash_map; //! @cond INTERNAL @@ -119,7 +122,7 @@ namespace interface5 { #endif //! Constructor hash_map_base() { - std::memset( (void*) this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 + std::memset( this, 0, pointers_per_table*sizeof(segment_ptr_t) // 32*4=128 or 64*8=512 + sizeof(my_size) + sizeof(my_mask) // 4+4 or 8+8 + embedded_buckets*sizeof(bucket) ); // n*8 or n*16 for( size_type i = 0; i < embedded_block; i++ ) // fill the table @@ -155,7 +158,7 @@ namespace interface5 { //! Initialize buckets static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) { - if( is_initial ) std::memset((void*) ptr, 0, sz*sizeof(bucket) ); + if( is_initial ) std::memset( static_cast(ptr), 0, sz*sizeof(bucket) ); else for(size_type i = 0; i < sz; i++, ptr++) { *reinterpret_cast(&ptr->mutex) = 0; ptr->node_list = rehash_req; @@ -179,22 +182,25 @@ namespace interface5 { }; //! Enable segment - void enable_segment( segment_index_t k, bool is_initial = false ) { + template + void enable_segment( segment_index_t k, const Allocator& allocator, bool is_initial = false ) { + typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; + typedef tbb::internal::allocator_traits bucket_allocator_traits; + bucket_allocator_type bucket_allocator(allocator); __TBB_ASSERT( k, "Zero segment must be embedded" ); enable_segment_failsafe watchdog( my_table, k ); - cache_aligned_allocator alloc; size_type sz; __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment"); if( k >= first_block ) { sz = segment_size( k ); - segment_ptr_t ptr = alloc.allocate( sz ); + segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz); init_buckets( ptr, sz, is_initial ); itt_hide_store_word( my_table[k], ptr ); sz <<= 1;// double it to get entire capacity of the container } else { // the first block __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); sz = segment_size( first_block ); - segment_ptr_t ptr = alloc.allocate( sz - embedded_buckets ); + segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz - embedded_buckets); init_buckets( ptr, sz - embedded_buckets, is_initial ); ptr -= segment_base(embedded_block); for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets @@ -204,6 +210,22 @@ namespace interface5 { watchdog.my_segment_ptr = 0; } + template + void delete_segment(segment_index_t s, const Allocator& allocator) { + typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; + typedef tbb::internal::allocator_traits bucket_allocator_traits; + bucket_allocator_type bucket_allocator(allocator); + segment_ptr_t buckets_ptr = my_table[s]; + size_type sz = segment_size( s ? s : 1 ); + + if( s >= first_block) // the first segment or the next + bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, sz); + else if( s == embedded_block && embedded_block != first_block ) + bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, + segment_size(first_block) - embedded_buckets); + if( s >= embedded_block ) my_table[s] = 0; + } + //! Get bucket by (masked) hashcode bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere? segment_index_t s = segment_index_of( h ); @@ -272,11 +294,12 @@ namespace interface5 { } //! Prepare enough segments for number of buckets - void reserve(size_type buckets) { + template + void reserve(size_type buckets, const Allocator& allocator) { if( !buckets-- ) return; bool is_initial = !my_size; for( size_type m = my_mask; buckets > m; m = my_mask ) - enable_segment( segment_index_of( m+1 ), is_initial ); + enable_segment( segment_index_of( m+1 ), allocator, is_initial ); } //! Swap hash_map_bases void internal_swap(hash_map_base &table) { @@ -288,6 +311,25 @@ namespace interface5 { for(size_type i = embedded_block; i < pointers_per_table; i++) swap(this->my_table[i], table.my_table[i]); } + +#if __TBB_CPP11_RVALUE_REF_PRESENT + void internal_move(hash_map_base&& other) { + my_mask = other.my_mask; + other.my_mask = embedded_buckets - 1; + my_size = other.my_size; + other.my_size = 0; + + for(size_type i = 0; i < embedded_buckets; ++i) { + my_embedded_segment[i].node_list = other.my_embedded_segment[i].node_list; + other.my_embedded_segment[i].node_list = NULL; + } + + for(size_type i = embedded_block; i < pointers_per_table; ++i) { + my_table[i] = other.my_table[i]; + other.my_table[i] = NULL; + } + } +#endif // __TBB_CPP11_RVALUE_REF_PRESENT }; template @@ -367,7 +409,7 @@ namespace interface5 { {} Value& operator*() const { __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); - return my_node->item; + return my_node->value(); } Value* operator->() const {return &operator*();} hash_map_iterator& operator++(); @@ -553,62 +595,78 @@ class concurrent_hash_map : protected internal::hash_map_base { protected: friend class const_accessor; - struct node; - typedef typename Allocator::template rebind::other node_allocator_type; + class node; + typedef typename tbb::internal::allocator_rebind::type node_allocator_type; + typedef tbb::internal::allocator_traits node_allocator_traits; node_allocator_type my_allocator; HashCompare my_hash_compare; - struct node : public node_base { - value_type item; - node( const Key &key ) : item(key, T()) {} - node( const Key &key, const T &t ) : item(key, t) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - node( const Key &key, T &&t ) : item(key, std::move(t)) {} - node( value_type&& i ) : item(std::move(i)){} -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - node( Args&&... args ) : item(std::forward(args)...) {} -#if __TBB_COPY_FROM_NON_CONST_REF_BROKEN - node( value_type& i ) : item(const_cast(i)) {} -#endif //__TBB_COPY_FROM_NON_CONST_REF_BROKEN -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - node( const value_type& i ) : item(i) {} - - // exception-safe allocation, see C++ Standard 2003, clause 5.3.4p17 - void *operator new( size_t /*size*/, node_allocator_type &a ) { - void *ptr = a.allocate(1); - if(!ptr) - tbb::internal::throw_exception(tbb::internal::eid_bad_alloc); - return ptr; - } - // match placement-new form above to be called if exception thrown in constructor - void operator delete( void *ptr, node_allocator_type &a ) { a.deallocate(static_cast(ptr),1); } + class node : public node_base { + tbb::aligned_space my_value; + public: + value_type* storage() { return my_value.begin(); } + value_type& value() { return *storage(); } }; void delete_node( node_base *n ) { - my_allocator.destroy( static_cast(n) ); - my_allocator.deallocate( static_cast(n), 1); + node_allocator_traits::destroy(my_allocator, static_cast(n)->storage()); + node_allocator_traits::destroy(my_allocator, static_cast(n)); + node_allocator_traits::deallocate(my_allocator, static_cast(n), 1); + } + + struct node_scoped_guard : tbb::internal::no_copy { + node* my_node; + node_allocator_type& my_alloc; + + node_scoped_guard(node* n, node_allocator_type& alloc) : my_node(n), my_alloc(alloc) {} + ~node_scoped_guard() { + if(my_node) { + node_allocator_traits::destroy(my_alloc, my_node); + node_allocator_traits::deallocate(my_alloc, my_node, 1); + } + } + void dismiss() { my_node = NULL; } + }; + +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + template + static node* create_node(node_allocator_type& allocator, Args&&... args) +#else + template + static node* create_node(node_allocator_type& allocator, __TBB_FORWARDING_REF(Arg1) arg1, __TBB_FORWARDING_REF(Arg2) arg2) +#endif + { + node* node_ptr = node_allocator_traits::allocate(allocator, 1); + node_scoped_guard guard(node_ptr, allocator); + node_allocator_traits::construct(allocator, node_ptr); +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + node_allocator_traits::construct(allocator, node_ptr->storage(), std::forward(args)...); +#else + node_allocator_traits::construct(allocator, node_ptr->storage(), tbb::internal::forward(arg1), tbb::internal::forward(arg2)); +#endif + guard.dismiss(); + return node_ptr; } static node* allocate_node_copy_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, *t); + return create_node(allocator, key, *t); } #if __TBB_CPP11_RVALUE_REF_PRESENT static node* allocate_node_move_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return new( allocator ) node(key, std::move(*const_cast(t))); + return create_node(allocator, key, std::move(*const_cast(t))); } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static node* allocate_node_emplace_construct(node_allocator_type& allocator, Args&&... args){ - return new( allocator ) node(std::forward(args)...); - } -#endif //#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif static node* allocate_node_default_construct(node_allocator_type& allocator, const Key &key, const T * ){ - return new( allocator ) node(key); +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT + // Emplace construct an empty T object inside the pair + return create_node(allocator, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); +#else + T obj; // Use of temporary object in impossible, because create_node takes non-const reference + return create_node(allocator, key, tbb::internal::move(obj)); +#endif } static node* do_not_allocate_node(node_allocator_type& , const Key &, const T * ){ @@ -618,7 +676,7 @@ class concurrent_hash_map : protected internal::hash_map_base { node *search_bucket( const key_type &key, bucket *b ) const { node *n = static_cast( b->node_list ); - while( is_valid(n) && !my_hash_compare.equal(key, n->item.first) ) + while( is_valid(n) && !my_hash_compare.equal(key, n->value().first) ) n = static_cast( n->next ); __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket"); return n; @@ -663,7 +721,7 @@ class concurrent_hash_map : protected internal::hash_map_base { __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL ); restart: for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(n)->item.first ); + hashcode_t c = my_hash_compare.hash( static_cast(n)->value().first ); #if TBB_USE_ASSERT hashcode_t bmask = h & (mask>>1); bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket @@ -715,7 +773,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Return reference to associated value in hash table. const_reference operator*() const { __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); - return my_node->item; + return my_node->value(); } //! Return pointer to associated value in hash table. @@ -745,7 +803,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Return reference to associated value in hash table. reference operator*() const { __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); - return this->my_node->item; + return this->my_node->value(); } //! Return pointer to associated value in hash table. @@ -759,18 +817,39 @@ class concurrent_hash_map : protected internal::hash_map_base { : internal::hash_map_base(), my_allocator(a) {} + explicit concurrent_hash_map( const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + {} + //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) { - reserve( n ); + reserve( n, my_allocator ); + } + + concurrent_hash_map( size_type n, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + { + reserve( n, my_allocator ); } //! Copy constructor - concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a = allocator_type() ) + concurrent_hash_map( const concurrent_hash_map &table ) + : internal::hash_map_base(), + my_allocator(node_allocator_traits::select_on_container_copy_construction(table.get_allocator())) + { + call_clear_on_leave scope_guard(this); + internal_copy(table); + scope_guard.dismiss(); + } + + concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a) : internal::hash_map_base(), my_allocator(a) { + call_clear_on_leave scope_guard(this); internal_copy(table); + scope_guard.dismiss(); } #if __TBB_CPP11_RVALUE_REF_PRESENT @@ -778,7 +857,7 @@ class concurrent_hash_map : protected internal::hash_map_base { concurrent_hash_map( concurrent_hash_map &&table ) : internal::hash_map_base(), my_allocator(std::move(table.get_allocator())) { - swap(table); + internal_move(std::move(table)); } //! Move constructor @@ -786,10 +865,10 @@ class concurrent_hash_map : protected internal::hash_map_base { : internal::hash_map_base(), my_allocator(a) { if (a == table.get_allocator()){ - this->swap(table); + internal_move(std::move(table)); }else{ call_clear_on_leave scope_guard(this); - internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end())); + internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end()), table.size()); scope_guard.dismiss(); } } @@ -798,19 +877,38 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Construction with copying iteration range and given allocator instance template concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) { - reserve( std::distance(first, last) ); // TODO: load_factor? - internal_copy(first, last); + call_clear_on_leave scope_guard(this); + internal_copy(first, last, std::distance(first, last)); + scope_guard.dismiss(); + } + + template + concurrent_hash_map( I first, I last, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + { + call_clear_on_leave scope_guard(this); + internal_copy(first, last, std::distance(first, last)); + scope_guard.dismiss(); } #if __TBB_INITIALIZER_LISTS_PRESENT //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. concurrent_hash_map( std::initializer_list il, const allocator_type &a = allocator_type() ) - : my_allocator(a) + : internal::hash_map_base(), my_allocator(a) { - reserve(il.size()); - internal_copy(il.begin(), il.end()); + call_clear_on_leave scope_guard(this); + internal_copy(il.begin(), il.end(), il.size()); + scope_guard.dismiss(); + } + + concurrent_hash_map( std::initializer_list il, const HashCompare& compare, const allocator_type& a = allocator_type() ) + : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) + { + call_clear_on_leave scope_guard(this); + internal_copy(il.begin(), il.end(), il.size()); + scope_guard.dismiss(); } #endif //__TBB_INITIALIZER_LISTS_PRESENT @@ -818,7 +916,9 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Assignment concurrent_hash_map& operator=( const concurrent_hash_map &table ) { if( this!=&table ) { + typedef typename node_allocator_traits::propagate_on_container_copy_assignment pocca_type; clear(); + tbb::internal::allocator_copy_assignment(my_allocator, table.my_allocator, pocca_type()); internal_copy(table); } return *this; @@ -827,17 +927,9 @@ class concurrent_hash_map : protected internal::hash_map_base { #if __TBB_CPP11_RVALUE_REF_PRESENT //! Move Assignment concurrent_hash_map& operator=( concurrent_hash_map &&table ) { - if(this != &table){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == table.my_allocator) { - concurrent_hash_map trash (std::move(*this)); - //TODO: swapping allocators here may be a problem, replace with single direction moving iff pocma is set - this->swap(table); - } else { - //do per element move - concurrent_hash_map moved_copy(std::move(table), this->my_allocator); - this->swap(moved_copy); - } + if(this != &table) { + typedef typename node_allocator_traits::propagate_on_container_move_assignment pocma_type; + internal_move_assign(std::move(table), pocma_type()); } return *this; } @@ -847,8 +939,7 @@ class concurrent_hash_map : protected internal::hash_map_base { //! Assignment concurrent_hash_map& operator=( std::initializer_list il ) { clear(); - reserve(il.size()); - internal_copy(il.begin(), il.end()); + internal_copy(il.begin(), il.end(), il.size()); return *this; } #endif //__TBB_INITIALIZER_LISTS_PRESENT @@ -1056,8 +1147,8 @@ class concurrent_hash_map : protected internal::hash_map_base { template bool generic_emplace( Accessor && result, Args &&... args ) { result.release(); - node * node_ptr = allocate_node_emplace_construct(my_allocator, std::forward(args)...); - return lookup(/*insert*/true, node_ptr->item.first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); + node * node_ptr = create_node(my_allocator, std::forward(args)...); + return lookup(/*insert*/true, node_ptr->value().first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); } #endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif //__TBB_CPP11_RVALUE_REF_PRESENT @@ -1073,7 +1164,24 @@ class concurrent_hash_map : protected internal::hash_map_base { void internal_copy( const concurrent_hash_map& source ); template - void internal_copy( I first, I last ); + void internal_copy( I first, I last, size_type reserve_size ); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + // A compile-time dispatch to allow move assignment of containers with non-movable value_type if POCMA is true_type + void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_true_type) { + tbb::internal::allocator_move_assignment(my_allocator, other.my_allocator, tbb::internal::traits_true_type()); + internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_false_type) { + if (this->my_allocator == other.my_allocator) { + internal_move(std::move(other)); + } else { + //do per element move + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), other.size()); + } + } +#endif //! Fast find when no concurrent erasure is used. For internal use inside TBB only! /** Return pointer to item with given key, or NULL if no such item exists. @@ -1105,6 +1213,33 @@ class concurrent_hash_map : protected internal::hash_map_base { } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +namespace internal { +using namespace tbb::internal; + +template typename Map, typename Key, typename T, typename... Args> +using hash_map_t = Map< + Key, T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash_compare >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > > +>; +} + +// Deduction guide for the constructor from two iterators and hash_compare/ allocator +template +concurrent_hash_map(I, I, Args...) +-> internal::hash_map_t,internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list and hash_compare/ allocator +// Deduction guide for an initializer_list, hash_compare and allocator is implicit +template +concurrent_hash_map(std::initializer_list>, CompareOrAllocator) +-> internal::hash_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template bool concurrent_hash_map::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key&, const T*), node *tmp_n ) { __TBB_ASSERT( !result || !result->my_node, NULL ); @@ -1177,7 +1312,7 @@ bool concurrent_hash_map::lookup( bool op_insert, const Key #if __TBB_STATISTICS my_info_resizes++; // concurrent ones #endif - enable_segment( grow_segment ); + enable_segment( grow_segment, my_allocator ); } if( tmp_n ) // if op_insert only delete_node( tmp_n ); @@ -1245,7 +1380,7 @@ bool concurrent_hash_map::erase( const Key &key ) { search: node_base **p = &b()->node_list; n = *p; - while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->item.first ) ) { + while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->value().first ) ) { p = &n->next; n = *p; } @@ -1272,16 +1407,18 @@ bool concurrent_hash_map::erase( const Key &key ) { template void concurrent_hash_map::swap(concurrent_hash_map &table) { - //TODO: respect C++11 allocator_traits::propogate_on_constainer_swap - using std::swap; - swap(this->my_allocator, table.my_allocator); - swap(this->my_hash_compare, table.my_hash_compare); - internal_swap(table); + typedef typename node_allocator_traits::propagate_on_container_swap pocs_type; + if (this != &table && (pocs_type::value || my_allocator == table.my_allocator)) { + using std::swap; + tbb::internal::allocator_swap(this->my_allocator, table.my_allocator, pocs_type()); + swap(this->my_hash_compare, table.my_hash_compare); + internal_swap(table); + } } template void concurrent_hash_map::rehash(size_type sz) { - reserve( sz ); // TODO: add reduction of number of buckets as well + reserve( sz, my_allocator ); // TODO: add reduction of number of buckets as well hashcode_t mask = my_mask; hashcode_t b = (mask+1)>>1; // size or first index of the last segment __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2 @@ -1300,7 +1437,7 @@ void concurrent_hash_map::rehash(size_type sz) { // now h - is index of the root rehashed bucket b_old mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(q)->item.first ); + hashcode_t c = my_hash_compare.hash( static_cast(q)->value().first ); if( (c & mask) != h ) { // should be rehashed *p = q->next; // exclude from b_old bucket *b_new = get_bucket( c & mask ); @@ -1327,7 +1464,7 @@ void concurrent_hash_map::rehash(size_type sz) { #endif #if TBB_USE_ASSERT for( ; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ) & mask; + hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ) & mask; __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" ); } #endif @@ -1374,7 +1511,7 @@ void concurrent_hash_map::clear() { #endif #if __TBB_EXTRA_DEBUG for(; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->item.first ); + hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ); h &= m; __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" ); } @@ -1404,11 +1541,10 @@ void concurrent_hash_map::clear() { reported = true; } #endif -#endif//TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS +#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS my_size = 0; segment_index_t s = segment_index_of( m ); __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" ); - cache_aligned_allocator alloc; do { __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" ); segment_ptr_t buckets_ptr = my_table[s]; @@ -1418,20 +1554,16 @@ void concurrent_hash_map::clear() { buckets_ptr[i].node_list = n->next; delete_node( n ); } - if( s >= first_block) // the first segment or the next - alloc.deallocate( buckets_ptr, sz ); - else if( s == embedded_block && embedded_block != first_block ) - alloc.deallocate( buckets_ptr, segment_size(first_block)-embedded_buckets ); - if( s >= embedded_block ) my_table[s] = 0; + delete_segment(s, my_allocator); } while(s-- > 0); my_mask = embedded_buckets - 1; } template void concurrent_hash_map::internal_copy( const concurrent_hash_map& source ) { - reserve( source.my_size ); // TODO: load_factor? hashcode_t mask = source.my_mask; if( my_mask == mask ) { // optimized version + reserve( source.my_size, my_allocator ); // TODO: load_factor? bucket *dst = 0, *src = 0; bool rehash_required = false; for( hashcode_t k = 0; k <= mask; k++ ) { @@ -1443,24 +1575,26 @@ void concurrent_hash_map::internal_copy( const concurrent_h rehash_required = true; dst->node_list = internal::rehash_req; } else for(; n; n = static_cast( n->next ) ) { - add_to_bucket( dst, new( my_allocator ) node(n->item.first, n->item.second) ); + node* node_ptr = create_node(my_allocator, n->value().first, n->value().second); + add_to_bucket( dst, node_ptr); ++my_size; // TODO: replace by non-atomic op } } if( rehash_required ) rehash(); - } else internal_copy( source.begin(), source.end() ); + } else internal_copy( source.begin(), source.end(), source.my_size ); } template template -void concurrent_hash_map::internal_copy(I first, I last) { +void concurrent_hash_map::internal_copy(I first, I last, size_type reserve_size) { + reserve( reserve_size, my_allocator ); // TODO: load_factor? hashcode_t m = my_mask; for(; first != last; ++first) { hashcode_t h = my_hash_compare.hash( (*first).first ); bucket *b = get_bucket( h & m ); __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = new( my_allocator ) node(*first); - add_to_bucket( b, n ); + node* node_ptr = create_node(my_allocator, (*first).first, (*first).second); + add_to_bucket( b, node_ptr ); ++my_size; // TODO: replace by non-atomic op } } diff --git a/src/tbb/include/tbb/concurrent_lru_cache.h b/src/tbb/include/tbb/concurrent_lru_cache.h index dc1532cf..c28fb6e2 100644 --- a/src/tbb/include/tbb/concurrent_lru_cache.h +++ b/src/tbb/include/tbb/concurrent_lru_cache.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_lru_cache_H diff --git a/src/tbb/include/tbb/concurrent_map.h b/src/tbb/include/tbb/concurrent_map.h new file mode 100644 index 00000000..d022d880 --- /dev/null +++ b/src/tbb/include/tbb/concurrent_map.h @@ -0,0 +1,383 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_map_H +#define __TBB_concurrent_map_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_map.h +#endif + +#include "tbb_config.h" + +// concurrent_map requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { + +namespace interface10 { + +template +class map_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using mapped_type = Value; + using compare_type = KeyCompare; + using value_type = std::pair; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + class value_compare { + public: + // TODO: these member types are deprecated in C++17, do we need to let them + using result_type = bool; + using first_argument_type = value_type; + using second_argument_type = value_type; + + bool operator()(const value_type& lhs, const value_type& rhs) const { + return comp(lhs.first, rhs.first); + } + + protected: + value_compare(compare_type c) : comp(c) {} + + friend class map_traits; + + compare_type comp; + }; + + static value_compare value_comp(compare_type comp) { return value_compare(comp); } + + static const key_type& get_key(const_reference val) { + return val.first; + } +}; // class map_traits + +template +class concurrent_multimap; + +template , typename Allocator = tbb_allocator>> +class concurrent_map + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = map_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_map() = default; + + explicit concurrent_map(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_map(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_map(const concurrent_map&) = default; + + concurrent_map(const concurrent_map& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_map(concurrent_map&&) = default; + + concurrent_map(concurrent_map&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_map(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_map(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_map& operator=(const concurrent_map& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_map& operator=(concurrent_map&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + mapped_type& at(const key_type& key) { + iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + const mapped_type& at(const key_type& key) const { + const_iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + mapped_type& operator[](const key_type& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + + return it->second; + } + + mapped_type& operator[](key_type&& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + + return it->second; + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Map, typename Key, typename T, typename... Args> +using c_map_t = Map 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > > >; +} // namespace internal + +template +concurrent_map(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_map(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator>> +class concurrent_multimap + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = map_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_multimap() = default; + + explicit concurrent_multimap(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multimap(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_multimap(const concurrent_multimap&) = default; + + concurrent_multimap(const concurrent_multimap& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multimap(concurrent_multimap&&) = default; + + concurrent_multimap(concurrent_multimap&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multimap(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multimap(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multimap& operator=(const concurrent_multimap& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multimap& operator=(concurrent_multimap&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + +}; // class concurrent_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +concurrent_multimap(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_multimap(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_map; +using interface10::concurrent_multimap; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_map_H diff --git a/src/tbb/include/tbb/concurrent_priority_queue.h b/src/tbb/include/tbb/concurrent_priority_queue.h index 89297d85..c5c03999 100644 --- a/src/tbb/include/tbb/concurrent_priority_queue.h +++ b/src/tbb/include/tbb/concurrent_priority_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_priority_queue_H @@ -27,6 +23,8 @@ #include "tbb_stddef.h" #include "tbb_profiling.h" #include "internal/_aggregator_impl.h" +#include "internal/_template_helpers.h" +#include "internal/_allocator_traits.h" #include #include #include @@ -85,14 +83,28 @@ class concurrent_priority_queue { typedef A allocator_type; //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), data(a) + explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(), data(a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + } + + //! Constructs a new concurrent_priority_queue with default capacity + explicit concurrent_priority_queue(const Compare& c, const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(c), data(a) { my_aggregator.initialize_handler(my_functor_t(this)); } //! Constructs a new concurrent_priority_queue with init_sz capacity explicit concurrent_priority_queue(size_type init_capacity, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), data(a) + mark(0), my_size(0), compare(), data(a) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(my_functor_t(this)); + } + + //! Constructs a new concurrent_priority_queue with init_sz capacity + explicit concurrent_priority_queue(size_type init_capacity, const Compare& c, const allocator_type& a = allocator_type()) : + mark(0), my_size(0), compare(c), data(a) { data.reserve(init_capacity); my_aggregator.initialize_handler(my_functor_t(this)); @@ -101,7 +113,17 @@ class concurrent_priority_queue { //! [begin,end) constructor template concurrent_priority_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - mark(0), data(begin, end, a) + mark(0), compare(), data(begin, end, a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + heapify(); + my_size = data.size(); + } + + //! [begin,end) constructor + template + concurrent_priority_queue(InputIterator begin, InputIterator end, const Compare& c, const allocator_type& a = allocator_type()) : + mark(0), compare(c), data(begin, end, a) { my_aggregator.initialize_handler(my_functor_t(this)); heapify(); @@ -111,7 +133,16 @@ class concurrent_priority_queue { #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from std::initializer_list concurrent_priority_queue(std::initializer_list init_list, const allocator_type &a = allocator_type()) : - mark(0),data(init_list.begin(), init_list.end(), a) + mark(0), compare(), data(init_list.begin(), init_list.end(), a) + { + my_aggregator.initialize_handler(my_functor_t(this)); + heapify(); + my_size = data.size(); + } + + //! Constructor from std::initializer_list + concurrent_priority_queue(std::initializer_list init_list, const Compare& c, const allocator_type &a = allocator_type()) : + mark(0), compare(c), data(init_list.begin(), init_list.end(), a) { my_aggregator.initialize_handler(my_functor_t(this)); heapify(); @@ -121,7 +152,7 @@ class concurrent_priority_queue { //! Copy constructor /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - explicit concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), + concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), my_size(src.my_size), data(src.data.begin(), src.data.end(), src.data.get_allocator()) { my_aggregator.initialize_handler(my_functor_t(this)); @@ -481,6 +512,31 @@ class concurrent_priority_queue { } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +namespace internal { + +template +using priority_queue_t = concurrent_priority_queue< + T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, cache_aligned_allocator > +>; +} + +// Deduction guide for the constructor from two iterators +template::value_type, + typename... Args +> concurrent_priority_queue(InputIterator, InputIterator, Args...) +-> internal::priority_queue_t; + +template +concurrent_priority_queue(std::initializer_list init_list, CompareOrAllocalor) +-> internal::priority_queue_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_priority_queue; diff --git a/src/tbb/include/tbb/concurrent_queue.h b/src/tbb/include/tbb/concurrent_queue.h index 81db58a3..be8e8f81 100644 --- a/src/tbb/include/tbb/concurrent_queue.h +++ b/src/tbb/include/tbb/concurrent_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_queue_H #define __TBB_concurrent_queue_H #include "internal/_concurrent_queue_impl.h" +#include "internal/_allocator_traits.h" namespace tbb { @@ -36,7 +33,7 @@ class concurrent_queue: public internal::concurrent_queue_base_v3 { template friend class internal::concurrent_queue_iterator; //! Allocator type - typedef typename A::template rebind::other page_allocator_type; + typedef typename tbb::internal::allocator_rebind::type page_allocator_type; page_allocator_type my_allocator; //! Allocates a block of size n (bytes) @@ -177,6 +174,15 @@ class concurrent_queue: public internal::concurrent_queue_base_v3 { const_iterator unsafe_end() const {return const_iterator();} } ; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template::value_type, + typename A = cache_aligned_allocator +> concurrent_queue(InputIterator, InputIterator, const A& = A()) +-> concurrent_queue; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template concurrent_queue::~concurrent_queue() { clear(); @@ -200,9 +206,9 @@ void concurrent_queue::clear() { template > class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { template friend class internal::concurrent_queue_iterator; + typedef typename tbb::internal::allocator_rebind::type page_allocator_type; //! Allocator type - typedef typename A::template rebind::other page_allocator_type; page_allocator_type my_allocator; typedef typename concurrent_queue_base_v3::padded_page padded_page; @@ -439,6 +445,15 @@ class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// guide for concurrent_bounded_queue(InputIterator, InputIterator, ...) +template::value_type, + typename A = cache_aligned_allocator +> concurrent_bounded_queue(InputIterator, InputIterator, const A& = A()) +-> concurrent_bounded_queue; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template concurrent_bounded_queue::~concurrent_bounded_queue() { clear(); diff --git a/src/tbb/include/tbb/concurrent_set.h b/src/tbb/include/tbb/concurrent_set.h new file mode 100644 index 00000000..70269947 --- /dev/null +++ b/src/tbb/include/tbb/concurrent_set.h @@ -0,0 +1,297 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_set_H +#define __TBB_concurrent_set_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_set.h +#endif + +#include "tbb/tbb_config.h" + +// concurrent_set requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { +namespace interface10 { + +// TODO: test this class +template +class set_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using value_type = key_type; + using compare_type = KeyCompare; + using value_compare = compare_type; + using reference = value_type & ; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + static const key_type& get_key(const_reference val) { + return val; + } + + static value_compare value_comp(compare_type comp) { return comp; } +}; + +template +class concurrent_multiset; + +template , typename Allocator = tbb_allocator> +class concurrent_set + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = set_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_set() = default; + + explicit concurrent_set(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_set(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_set(const concurrent_set&) = default; + + concurrent_set(const concurrent_set& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_set(concurrent_set&&) = default; + + concurrent_set(concurrent_set&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_set(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_set(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_set& operator=(const concurrent_set& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_set& operator=(concurrent_set&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Set, typename Key, typename... Args> +using c_set_t = Set 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > >; +} // namespace internal + +template +concurrent_set(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_set(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator> +class concurrent_multiset + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = set_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_multiset() = default; + + explicit concurrent_multiset(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multiset(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(first, last); + } + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const allocator_type& alloc) : base_type(key_compare(), alloc) { + insert(first, last); + } + + /** Copy constructor */ + concurrent_multiset(const concurrent_multiset&) = default; + + concurrent_multiset(const concurrent_multiset& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multiset(concurrent_multiset&&) = default; + + concurrent_multiset(concurrent_multiset&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multiset(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multiset(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multiset& operator=(const concurrent_multiset& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multiset& operator=(concurrent_multiset&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + + +template +concurrent_multiset(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_multiset(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_set; +using interface10::concurrent_multiset; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_set_H diff --git a/src/tbb/include/tbb/concurrent_unordered_map.h b/src/tbb/include/tbb/concurrent_unordered_map.h index c959a7ec..cc73dad7 100644 --- a/src/tbb/include/tbb/concurrent_unordered_map.h +++ b/src/tbb/include/tbb/concurrent_unordered_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -39,7 +35,13 @@ class concurrent_unordered_map_traits typedef std::pair value_type; typedef Key key_type; typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef tbb::internal::node_handle::node, + allocator_type> node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + enum { allow_multimapping = Allow_multimapping }; concurrent_unordered_map_traits() : my_hash_compare() {} @@ -53,6 +55,9 @@ class concurrent_unordered_map_traits hash_compare my_hash_compare; // the comparator predicate for keys }; +template +class concurrent_unordered_multimap; + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > > class concurrent_unordered_map : @@ -93,12 +98,23 @@ class concurrent_unordered_map : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_map(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_map(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_map(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_map(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -106,9 +122,24 @@ class concurrent_unordered_map : template concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } @@ -116,16 +147,30 @@ class concurrent_unordered_map : #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(il.begin(),il.end()); + } + + concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(), il.end()); } + + concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_map(const concurrent_unordered_map& table) : base_type(table) {} @@ -143,11 +188,31 @@ class concurrent_unordered_map : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_map(concurrent_unordered_map&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_map& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_map&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_map(const concurrent_unordered_map& table, const Allocator& a) : base_type(table, a) @@ -191,6 +256,45 @@ class concurrent_unordered_map : } }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { +using namespace tbb::internal; + +template typename Map, typename Key, typename Element, typename... Args> +using cu_map_t = Map< + Key, Element, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash >, + std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, + pack_element_t<1, Args...>, std::equal_to >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > > +>; +} + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_map (I, I) +-> internal::cu_map_t, internal::iterator_mapped_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_map(I, I, size_t, Args...) +-> internal::cu_map_t, internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_map(std::initializer_list>) +-> internal::cu_map_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_map(std::initializer_list>, size_t, Args...) +-> internal::cu_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template < typename Key, typename T, typename Hasher = tbb::tbb_hash, typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > > class concurrent_unordered_multimap : @@ -229,12 +333,23 @@ class concurrent_unordered_multimap : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_multimap(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_multimap(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_multimap(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_multimap(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -242,9 +357,24 @@ class concurrent_unordered_multimap : template concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets,key_compare(_Hasher,_Key_equality), a) + : base_type(n_of_buckets,key_compare(a_hasher,a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } @@ -252,16 +382,29 @@ class concurrent_unordered_multimap : #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(il.begin(),il.end()); + } + + concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(), il.end()); } + + concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_multimap(const concurrent_unordered_multimap& table) : base_type(table) {} @@ -279,16 +422,60 @@ class concurrent_unordered_multimap : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_multimap(concurrent_unordered_multimap&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_map& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_map&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multimap&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_multimap(const concurrent_unordered_multimap& table, const Allocator& a) : base_type(table, a) {} }; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_multimap (I, I) +-> internal::cu_map_t, internal::iterator_mapped_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_multimap(I, I, size_t, Args...) +-> internal::cu_map_t, internal::iterator_mapped_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_multimap(std::initializer_list>) +-> internal::cu_map_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_multimap(std::initializer_list>, size_t, Args...) +-> internal::cu_map_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_unordered_map; diff --git a/src/tbb/include/tbb/concurrent_unordered_set.h b/src/tbb/include/tbb/concurrent_unordered_set.h index a26fee1a..93a772ba 100644 --- a/src/tbb/include/tbb/concurrent_unordered_set.h +++ b/src/tbb/include/tbb/concurrent_unordered_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -39,7 +35,13 @@ class concurrent_unordered_set_traits typedef Key value_type; typedef Key key_type; typedef Hash_compare hash_compare; - typedef typename Allocator::template rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef tbb::internal::node_handle::node, + allocator_type> node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + enum { allow_multimapping = Allow_multimapping }; concurrent_unordered_set_traits() : my_hash_compare() {} @@ -52,6 +54,9 @@ class concurrent_unordered_set_traits hash_compare my_hash_compare; // the comparator predicate for keys }; +template +class concurrent_unordered_multiset; + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > class concurrent_unordered_set : public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, false> > { @@ -87,6 +92,9 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif /*__TBB_UNORDERED_NODE_HANDLE_PRESENT*/ // Construction/destruction/copying explicit concurrent_unordered_set(size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), @@ -94,29 +102,63 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) {} + concurrent_unordered_set(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_set(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + {} + explicit concurrent_unordered_set(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) {} template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { insert(first, last); } + template + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(first, last); + } + #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(),il.end()); + } + + concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(il.begin(), il.end()); } + + concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_set(const concurrent_unordered_set& table) : base_type(table) {} @@ -134,12 +176,32 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_set(concurrent_unordered_set&& table, const Allocator& a) : base_type(std::move(table), a) {} -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_set& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_set&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_set(const concurrent_unordered_set& table, const Allocator& a) : base_type(table, a) @@ -147,6 +209,45 @@ class concurrent_unordered_set : public internal::concurrent_unordered_base< con }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { +using namespace tbb::internal; + +template typename Set, typename T, typename... Args> +using cu_set_t = Set < + T, + std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, + pack_element_t<0, Args...>, tbb_hash >, + std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, + pack_element_t<1, Args...>, std::equal_to >, + std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, + pack_element_t, tbb_allocator > +>; +} + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_set(I, I) +-> internal::cu_set_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_set(I, I, size_t, Args...) +-> internal::cu_set_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_set(std::initializer_list) +-> internal::cu_set_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_set(std::initializer_list, size_t, Args...) +-> internal::cu_set_t; + +#endif /*__TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > class concurrent_unordered_multiset : @@ -185,12 +286,24 @@ class concurrent_unordered_multiset : typedef typename base_type::const_iterator const_iterator; typedef typename base_type::iterator local_iterator; typedef typename base_type::const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename base_type::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Construction/destruction/copying explicit concurrent_unordered_multiset(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + {} + + concurrent_unordered_multiset(size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + {} + + concurrent_unordered_multiset(size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) {} explicit concurrent_unordered_multiset(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) @@ -198,25 +311,54 @@ class concurrent_unordered_multiset : template concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& _Hasher = hasher(), const key_equal& _Key_equality = key_equal(), + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(_Hasher, _Key_equality), a) + : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) + { + insert(first, last); + } + + template + concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(first, last); + } + + template + concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) { insert(first, last); } #if __TBB_INITIALIZER_LISTS_PRESENT //! Constructor from initializer_list - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, + const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) { - this->insert(il.begin(),il.end()); + insert(il.begin(),il.end()); + } + + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) + : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) + { + insert(il.begin(), il.end()); } + + concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, + const allocator_type& a) + : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) + { + insert(il.begin(), il.end()); + } + #endif //# __TBB_INITIALIZER_LISTS_PRESENT -#if __TBB_CPP11_RVALUE_REF_PRESENT -#if !__TBB_IMPLICIT_MOVE_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT concurrent_unordered_multiset(const concurrent_unordered_multiset& table) : base_type(table) {} @@ -234,18 +376,62 @@ class concurrent_unordered_multiset : { return static_cast(base_type::operator=(std::move(table))); } -#endif //!__TBB_IMPLICIT_MOVE_PRESENT +#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT +#if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_multiset(concurrent_unordered_multiset&& table, const Allocator& a) : base_type(std::move(table), a) { } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void merge(concurrent_unordered_set& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_set&& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset& source) + { this->internal_merge(source); } + + template + void merge(concurrent_unordered_multiset&& source) + { this->internal_merge(source); } + +#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT concurrent_unordered_multiset(const concurrent_unordered_multiset& table, const Allocator& a) : base_type(table, a) {} }; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +// Deduction guide for the constructor from two iterators +template +concurrent_unordered_multiset(I, I) +-> internal::cu_set_t>; + +// Deduction guide for the constructor from two iterators and hasher/equality/allocator +template +concurrent_unordered_multiset(I, I, size_t, Args...) +-> internal::cu_set_t, Args...>; + +// Deduction guide for the constructor from an initializer_list +template +concurrent_unordered_multiset(std::initializer_list) +-> internal::cu_set_t; + +// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator +template +concurrent_unordered_multiset(std::initializer_list, size_t, Args...) +-> internal::cu_set_t; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ } // namespace interface5 using interface5::concurrent_unordered_set; diff --git a/src/tbb/include/tbb/concurrent_vector.h b/src/tbb/include/tbb/concurrent_vector.h index b9bd5559..461d4c35 100644 --- a/src/tbb/include/tbb/concurrent_vector.h +++ b/src/tbb/include/tbb/concurrent_vector.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_vector_H @@ -34,6 +30,8 @@ #include #include +#include "internal/_allocator_traits.h" + #if _MSC_VER==1500 && !__INTEL_COMPILER // VS2008/VC9 seems to have an issue; limits pull in math.h #pragma warning( push ) @@ -74,7 +72,7 @@ namespace internal { //! Exception helper function template void handle_unconstructed_elements(T* array, size_t n_of_elements){ - std::memset((void*) array, 0, n_of_elements * sizeof( T )); + std::memset( static_cast(array), 0, n_of_elements * sizeof( T ) ); } //! Base class of concurrent vector implementation. @@ -94,11 +92,11 @@ namespace internal { pointers_per_short_table = 3, // to fit into 8 words of entire structure pointers_per_long_table = sizeof(segment_index_t) * 8 // one segment per bit }; - public: + struct segment_not_used {}; struct segment_allocated {}; struct segment_allocation_failed {}; - protected: + class segment_t; class segment_value_t { void* array; @@ -467,12 +465,9 @@ namespace internal { template class allocator_base { public: - typedef typename A::template - rebind::other allocator_type; + typedef typename tbb::internal::allocator_rebind::type allocator_type; allocator_type my_allocator; - allocator_base(const allocator_type &a = allocator_type() ) : my_allocator(a) {} - }; } // namespace internal @@ -742,9 +737,7 @@ class concurrent_vector: protected internal::allocator_base, if(pocma_t::value || this->my_allocator == other.my_allocator) { concurrent_vector trash (std::move(*this)); internal_swap(other); - if (pocma_t::value) { - this->my_allocator = std::move(other.my_allocator); - } + tbb::internal::allocator_move_assignment(this->my_allocator, other.my_allocator, pocma_t()); } else { internal_assign(other, sizeof(T), &destroy_array, &move_assign_array, &move_array); } @@ -1002,10 +995,10 @@ class concurrent_vector: protected internal::allocator_base, //! swap two instances void swap(concurrent_vector &vector) { - using std::swap; - if( this != &vector ) { + typedef typename tbb::internal::allocator_traits::propagate_on_container_swap pocs_t; + if( this != &vector && (this->my_allocator == vector.my_allocator || pocs_t::value) ) { concurrent_vector_base_v3::internal_swap(static_cast(vector)); - swap(this->my_allocator, vector.my_allocator); + tbb::internal::allocator_swap(this->my_allocator, vector.my_allocator, pocs_t()); } } @@ -1042,7 +1035,13 @@ class concurrent_vector: protected internal::allocator_base, internal_resize( n, sizeof(T), max_size(), static_cast(p), &destroy_array, p? &initialize_array_by : &initialize_array ); } - //! helper class + //! True/false function override helper + /* Functions declarations: + * void foo(is_integer_tag*); + * void foo(is_integer_tag*); + * Usage example: + * foo(static_cast::is_integer>*>(0)); + */ template class is_integer_tag; //! assign integer items by copying when arguments are treated as iterators. See C++ Standard 2003 23.1.1p9 @@ -1156,6 +1155,25 @@ class concurrent_vector: protected internal::allocator_base, }; }; +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template::value_type, + typename A = cache_aligned_allocator +> concurrent_vector(I, I, const A& = A()) +-> concurrent_vector; + +// Deduction guide for the constructor from a vector and allocator +template +concurrent_vector(const concurrent_vector &, const A2 &) +-> concurrent_vector; + +// Deduction guide for the constructor from an initializer_list +template +> concurrent_vector(std::initializer_list, const A& = A()) +-> concurrent_vector; +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) #pragma warning (push) #pragma warning (disable: 4701) // potentially uninitialized local variable "old" @@ -1293,8 +1311,8 @@ void concurrent_vector::move_array_if_noexcept( void* dst, const void* src template template void concurrent_vector::copy_range( void* dst, const void* p_type_erased_iterator, size_type n ){ - I & iterator ((*const_cast(static_cast(p_type_erased_iterator)))); - internal_loop_guide loop(n, dst); loop.iterate(iterator); + internal_loop_guide loop(n, dst); + loop.iterate( *(static_cast(const_cast(p_type_erased_iterator))) ); } template diff --git a/src/tbb/include/tbb/critical_section.h b/src/tbb/include/tbb/critical_section.h index 324b3e13..ba693b38 100644 --- a/src/tbb/include/tbb/critical_section.h +++ b/src/tbb/include/tbb/critical_section.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_CRITICAL_SECTION_H_ diff --git a/src/tbb/include/tbb/enumerable_thread_specific.h b/src/tbb/include/tbb/enumerable_thread_specific.h index f31f533b..1bcac28d 100644 --- a/src/tbb/include/tbb/enumerable_thread_specific.h +++ b/src/tbb/include/tbb/enumerable_thread_specific.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_enumerable_thread_specific_H diff --git a/src/tbb/include/tbb/flow_graph.h b/src/tbb/include/tbb/flow_graph.h index fb839ac1..cbdad306 100644 --- a/src/tbb/include/tbb/flow_graph.h +++ b/src/tbb/include/tbb/flow_graph.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_H @@ -34,7 +30,6 @@ #include "internal/_aggregator_impl.h" #include "tbb_profiling.h" #include "task_arena.h" -#include "flow_graph_abstractions.h" #if __TBB_PREVIEW_ASYNC_MSG #include // std::vector in internal::async_storage @@ -100,7 +95,11 @@ class continue_msg {}; template< typename T > class sender; template< typename T > class receiver; class continue_receiver; -template< typename T > class limiter_node; // needed for resetting decrementer +} // namespaceX +namespace interface11 { +template< typename T, typename U > class limiter_node; // needed for resetting decrementer +} +namespace interface10 { template< typename R, typename B > class run_and_put_task; namespace internal { @@ -111,7 +110,7 @@ template class round_robin_cache; template class predecessor_cache; template class reservable_predecessor_cache; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // Holder of edges both for caches and for those nodes which do not have predecessor caches. // C == receiver< ... > or sender< ... >, depending. template @@ -153,7 +152,7 @@ class edge_container { private: edge_list_type built_edges; }; // class edge_container -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } // namespace internal @@ -301,7 +300,7 @@ class untyped_sender { //! Consumes the reserved item virtual bool try_consume( ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef internal::edge_container built_successors_type; typedef built_successors_type::edge_list_type successor_list_type; @@ -310,7 +309,7 @@ class untyped_sender { virtual void internal_delete_built_successor( successor_type & ) = 0; virtual void copy_successors( successor_list_type &) = 0; virtual size_t successor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: //! Request an item from the sender template< typename X > @@ -330,7 +329,6 @@ class untyped_sender { class untyped_receiver { template< typename, typename > friend class run_and_put_task; - template< typename > friend class limiter_node; template< typename, typename > friend class internal::broadcast_cache; template< typename, typename > friend class internal::round_robin_cache; @@ -365,7 +363,7 @@ class untyped_receiver { //! Remove a predecessor from the node virtual bool remove_predecessor( predecessor_type & ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef internal::edge_container built_predecessors_type; typedef built_predecessors_type::edge_list_type predecessor_list_type; virtual built_predecessors_type &built_predecessors() = 0; @@ -373,7 +371,7 @@ class untyped_receiver { virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; virtual void copy_predecessors( predecessor_list_type & ) = 0; virtual size_t predecessor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template task *try_put_task(const X& t) { @@ -495,7 +493,7 @@ class sender { //! Consumes the reserved item virtual bool try_consume( ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef typename internal::edge_container built_successors_type; typedef typename built_successors_type::edge_list_type successor_list_type; @@ -504,7 +502,7 @@ class sender { virtual void internal_delete_built_successor( successor_type & ) = 0; virtual void copy_successors( successor_list_type &) = 0; virtual size_t successor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ }; // class sender //! Pure virtual template class that defines a receiver of messages of type T @@ -544,7 +542,7 @@ class receiver { //! Remove a predecessor from the node virtual bool remove_predecessor( predecessor_type & ) { return false; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename internal::edge_container built_predecessors_type; typedef typename built_predecessors_type::edge_list_type predecessor_list_type; virtual built_predecessors_type &built_predecessors() = 0; @@ -552,11 +550,10 @@ class receiver { virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; virtual void copy_predecessors( predecessor_list_type & ) = 0; virtual size_t predecessor_count() = 0; -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: //! put receiver back in initial state - template friend class limiter_node; virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; template friend class internal::successor_cache; @@ -581,15 +578,18 @@ class continue_receiver : public receiver< continue_msg > { typedef receiver::predecessor_type predecessor_type; //! Constructor - explicit continue_receiver( int number_of_predecessors = 0 ) { + explicit continue_receiver( + __TBB_FLOW_GRAPH_PRIORITY_ARG1(int number_of_predecessors, node_priority_t priority)) { my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; my_current_count = 0; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = priority; ) } //! Copy constructor continue_receiver( const continue_receiver& src ) : receiver() { my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; my_current_count = 0; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = src.my_priority; ) } //! Increments the trigger threshold @@ -609,7 +609,7 @@ class continue_receiver : public receiver< continue_msg > { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef internal::edge_container built_predecessors_type; typedef built_predecessors_type::edge_list_type predecessor_list_type; built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } @@ -634,7 +634,7 @@ class continue_receiver : public receiver< continue_msg > { return my_built_predecessors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template< typename R, typename B > friend class run_and_put_task; @@ -653,7 +653,7 @@ class continue_receiver : public receiver< continue_msg > { return res? res : SUCCESSFULLY_ENQUEUED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // continue_receiver must contain its own built_predecessors because it does // not have a node_cache. built_predecessors_type my_built_predecessors; @@ -662,14 +662,15 @@ class continue_receiver : public receiver< continue_msg > { int my_predecessor_count; int my_current_count; int my_initial_predecessor_count; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) // the friend declaration in the base class did not eliminate the "protected class" // error in gcc 4.1.2 - template friend class limiter_node; + template friend class tbb::flow::interface11::limiter_node; void reset_receiver( reset_flags f ) __TBB_override { my_current_count = 0; if (f & rf_clear_edges) { -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_predecessors.clear(); #endif my_predecessor_count = my_initial_predecessor_count; @@ -744,7 +745,7 @@ inline graph::graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) own_context = true; cancelled = false; caught_exception = false; - my_context = new task_group_context(); + my_context = new task_group_context(tbb::internal::FLOW_TASKS); my_root_task = (new (task::allocate_root(*my_context)) empty_task); my_root_task->set_ref_count(1); tbb::internal::fgt_graph(this); @@ -761,7 +762,15 @@ inline graph::graph(task_group_context& use_this_context) : my_is_active = true; } -inline void graph::reserve_wait() { +inline graph::~graph() { + wait_for_all(); + my_root_task->set_ref_count(0); + tbb::task::destroy(*my_root_task); + if (own_context) delete my_context; + delete my_task_arena; +} + +inline void graph::reserve_wait() { if (my_root_task) { my_root_task->increment_ref_count(); tbb::internal::fgt_reserve_wait(this); @@ -816,17 +825,37 @@ inline void graph::reset( reset_flags f ) { internal::activate_graph(*this); // now spawn the tasks necessary to start the graph for(task_list_type::iterator rti = my_reset_task_list.begin(); rti != my_reset_task_list.end(); ++rti) { - my_task_arena->execute(graph::spawn_functor(*(*rti))); + internal::spawn_in_graph_arena(*this, *(*rti)); } my_reset_task_list.clear(); } +inline graph::iterator graph::begin() { return iterator(this, true); } + +inline graph::iterator graph::end() { return iterator(this, false); } + +inline graph::const_iterator graph::begin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::end() const { return const_iterator(this, false); } + +inline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::cend() const { return const_iterator(this, false); } + #if TBB_PREVIEW_FLOW_GRAPH_TRACE inline void graph::set_name(const char *name) { tbb::internal::fgt_graph_desc(this, name); } #endif +inline graph_node::graph_node(graph& g) : my_graph(g) { + my_graph.register_node(this); +} + +inline graph_node::~graph_node() { + my_graph.remove_node(this); +} + #include "internal/_flow_graph_node_impl.h" //! An executable node that acts as a source, i.e. it has no predecessors @@ -842,7 +871,7 @@ class source_node : public graph_node, public sender< Output > { //Source node has no input type typedef null_type input_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -897,7 +926,7 @@ class source_node : public graph_node, public sender< Output > { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -920,7 +949,7 @@ class source_node : public graph_node, public sender< Output > { spin_mutex::scoped_lock l(my_mutex); my_successors.copy_successors(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Request an item from the node bool try_get( output_type &v ) __TBB_override { @@ -992,7 +1021,7 @@ class source_node : public graph_node, public sender< Output > { return dynamic_cast< internal::source_body_leaf & >(body_ref).get_body(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) __TBB_override { my_successors.built_successors().sender_extract(*this); // removes "my_owner" == this from each successor my_active = init_my_active; @@ -1086,41 +1115,32 @@ class source_node : public graph_node, public sender< Output > { } }; // class source_node -template -struct allocate_buffer { - static const bool value = false; -}; - -template<> -struct allocate_buffer { - static const bool value = true; -}; - //! Implements a function node that supports Input -> Output template < typename Input, typename Output = continue_msg, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class function_node : public graph_node, public internal::function_input, public internal::function_output { +class function_node : public graph_node, public internal::function_input, public internal::function_output { public: typedef Input input_type; typedef Output output_type; - typedef internal::function_input fInput_type; + typedef internal::function_input input_impl_type; typedef internal::function_input_queue input_queue_type; typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; + typedef typename input_impl_type::predecessor_type predecessor_type; typedef typename fOutput_type::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename fInput_type::predecessor_list_type predecessor_list_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename input_impl_type::predecessor_list_type predecessor_list_type; typedef typename fOutput_type::successor_list_type successor_list_type; #endif - using fInput_type::my_predecessors; + using input_impl_type::my_predecessors; //! Constructor // input_queue_type is allocated here, but destroyed in the function_input_base. // TODO: pass the graph_buffer_policy to the function_input_base so it can all // be done in one place. This would be an interface-breaking change. template< typename Body > - function_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), fInput_type(g, concurrency, body, allocate_buffer::value ? - new input_queue_type( ) : NULL ) { + function_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), input_impl_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); } @@ -1128,7 +1148,7 @@ class function_node : public graph_node, public internal::function_input::value ? new input_queue_type : NULL), + input_impl_type(src), fOutput_type() { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); @@ -1140,7 +1160,7 @@ class function_node : public graph_node, public internal::function_input friend class run_and_put_task; template friend class internal::broadcast_cache; template friend class internal::round_robin_cache; - using fInput_type::try_put_task; + using input_impl_type::try_put_task; internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_function_input(f); + input_impl_type::reset_function_input(f); // TODO: use clear() instead. if(f & rf_clear_edges) { successors().clear(); @@ -1181,6 +1201,7 @@ class multifunction_node : internal::multifunction_output, // wrap this around each element Output // the tuple providing the types >::type, + Policy, Allocator > { protected: @@ -1189,22 +1210,26 @@ class multifunction_node : typedef Input input_type; typedef null_type output_type; typedef typename internal::wrap_tuple_elements::type output_ports_type; - typedef internal::multifunction_input fInput_type; + typedef internal::multifunction_input input_impl_type; typedef internal::function_input_queue input_queue_type; private: - typedef typename internal::multifunction_input base_type; - using fInput_type::my_predecessors; + typedef typename internal::multifunction_input base_type; + using input_impl_type::my_predecessors; public: template - multifunction_node( graph &g, size_t concurrency, Body body ) : - graph_node(g), base_type(g,concurrency, body, allocate_buffer::value ? new input_queue_type : NULL) { - tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); + multifunction_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), base_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { + tbb::internal::fgt_multioutput_node_with_body( + tbb::internal::FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); } multifunction_node( const multifunction_node &other) : - graph_node(other.my_graph), base_type(other, allocate_buffer::value ? new input_queue_type : NULL) { + graph_node(other.my_graph), base_type(other) { tbb::internal::fgt_multioutput_node_with_body( tbb::internal::FLOW_MULTIFUNCTION_NODE, &this->my_graph, static_cast *>(this), this->output_ports(), this->my_body ); @@ -1216,7 +1241,7 @@ class multifunction_node : } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) __TBB_override { my_predecessors.built_predecessors().receiver_extract(*this); base_type::extract(); @@ -1236,7 +1261,7 @@ class split_node : public graph_node, public receiver { public: typedef TupleType input_type; typedef Allocator allocator_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename base_type::predecessor_type predecessor_type; typedef typename base_type::predecessor_list_type predecessor_list_type; typedef internal::predecessor_cache predecessor_cache_type; @@ -1284,7 +1309,7 @@ class split_node : public graph_node, public receiver { graph& graph_reference() __TBB_override { return my_graph; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION private: //! split_node doesn't use this "predecessors" functionality; so, we have "dummies" here; void extract() __TBB_override {} @@ -1302,37 +1327,42 @@ class split_node : public graph_node, public receiver { //! dummy member built_predecessors_type my_predessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ private: output_ports_type my_output_ports; }; //! Implements an executable node that supports continue_msg -> Output -template -class continue_node : public graph_node, public internal::continue_input, public internal::function_output { +template > +class continue_node : public graph_node, public internal::continue_input, + public internal::function_output { public: typedef continue_msg input_type; typedef Output output_type; - typedef internal::continue_input fInput_type; + typedef internal::continue_input input_impl_type; typedef internal::function_output fOutput_type; - typedef typename fInput_type::predecessor_type predecessor_type; + typedef typename input_impl_type::predecessor_type predecessor_type; typedef typename fOutput_type::successor_type successor_type; //! Constructor for executable node with continue_msg -> Output template - continue_node( graph &g, Body body ) : - graph_node(g), internal::continue_input( g, body ) { + continue_node( + graph &g, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g), input_impl_type( g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority) ) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); } - //! Constructor for executable node with continue_msg -> Output template - continue_node( graph &g, int number_of_predecessors, Body body ) : - graph_node(g), internal::continue_input( g, number_of_predecessors, body ) { + continue_node( + graph &g, int number_of_predecessors, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : graph_node(g) + , input_impl_type(g, number_of_predecessors, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this), this->my_body ); @@ -1340,7 +1370,7 @@ class continue_node : public graph_node, public internal::continue_input //! Copy constructor continue_node( const continue_node& src ) : - graph_node(src.my_graph), internal::continue_input(src), + graph_node(src.my_graph), input_impl_type(src), internal::function_output() { tbb::internal::fgt_node_with_body( tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, static_cast *>(this), @@ -1353,9 +1383,9 @@ class continue_node : public graph_node, public internal::continue_input } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { - fInput_type::my_built_predecessors.receiver_extract(*this); + input_impl_type::my_built_predecessors.receiver_extract(*this); successors().built_successors().sender_extract(*this); } #endif @@ -1364,430 +1394,173 @@ class continue_node : public graph_node, public internal::continue_input template< typename R, typename B > friend class run_and_put_task; template friend class internal::broadcast_cache; template friend class internal::round_robin_cache; - using fInput_type::try_put_task; + using input_impl_type::try_put_task; internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } void reset_node(reset_flags f) __TBB_override { - fInput_type::reset_receiver(f); + input_impl_type::reset_receiver(f); if(f & rf_clear_edges)successors().clear(); __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); } }; // continue_node -template< typename T > -class overwrite_node : public graph_node, public receiver, public sender { +//! Forwards messages of type T to all successors +template +class broadcast_node : public graph_node, public receiver, public sender { public: typedef T input_type; typedef T output_type; typedef typename receiver::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename sender::successor_list_type successor_list_type; #endif +private: + internal::broadcast_cache my_successors; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + internal::edge_container my_built_predecessors; + spin_mutex pred_mutex; // serialize accesses on edge_container +#endif +public: - explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { + explicit broadcast_node(graph& g) : graph_node(g) { my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this) ); } - //! Copy constructor; doesn't take anything from src; default won't work - overwrite_node( const overwrite_node& src ) : - graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) + // Copy constructor + broadcast_node( const broadcast_node& src ) : + graph_node(src.my_graph), receiver(), sender() { my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, static_cast *>(this), static_cast *>(this) ); } - ~overwrite_node() {} - #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { tbb::internal::fgt_node_desc( this, name ); } #endif - bool register_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if (my_buffer_is_valid && internal::is_graph_active( my_graph )) { - // We have a valid value that must be forwarded immediately. - bool ret = s.try_put( my_buffer ); -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - if ( ret ) { - // We add the successor that accepted our put - my_successors.register_successor( s ); - } else { - // In case of reservation a race between the moment of reservation and register_successor can appear, - // because failed reserve does not mean that register_successor is not ready to put a message immediately. - // We have some sort of infinite loop: reserving node tries to set pull state for the edge, - // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. - task *rtask = new ( task::allocate_additional_child_of( *( my_graph.root_task() ) ) ) - register_predecessor_task( *this, s ); - internal::spawn_in_graph_arena( my_graph, *rtask ); - } -#else - if ( ret || !s.register_predecessor( *this ) ) { - // We add the successor: it accepted our put or it rejected it but won't let us become a predecessor - my_successors.register_successor( s ); - } else { - // We don't add the successor: it rejected our put and we became its predecessor instead - return false; - } -#endif - } else { - // No valid value yet, just add as successor - my_successors.register_successor( s ); - } + //! Adds a successor + bool register_successor( successor_type &r ) __TBB_override { + my_successors.register_successor( r ); return true; } - bool remove_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.remove_successor(s); + //! Removes s as a successor + bool remove_successor( successor_type &r ) __TBB_override { + my_successors.remove_successor( r ); return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename sender::built_successors_type built_successors_type; - void internal_add_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_add_built_successor(s); + built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } + + void internal_add_built_successor(successor_type &r) __TBB_override { + my_successors.internal_add_built_successor(r); } - void internal_delete_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_delete_built_successor(s); + void internal_delete_built_successor(successor_type &r) __TBB_override { + my_successors.internal_delete_built_successor(r); } size_t successor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); return my_successors.successor_count(); } void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); my_successors.copy_successors(v); } + typedef typename receiver::built_predecessors_type built_predecessors_type; + + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); + spin_mutex::scoped_lock l(pred_mutex); my_built_predecessors.add_edge(p); } void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); + spin_mutex::scoped_lock l(pred_mutex); my_built_predecessors.delete_edge(p); } size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); + spin_mutex::scoped_lock l(pred_mutex); return my_built_predecessors.edge_count(); } - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); + void copy_predecessors(predecessor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l(pred_mutex); my_built_predecessors.copy_edges(v); } void extract() __TBB_override { - my_buffer_is_valid = false; - built_successors().sender_extract(*this); - built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - - bool try_get( input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if ( my_buffer_is_valid ) { - v = my_buffer; - return true; - } - return false; - } - -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - //! Reserves an item - bool try_reserve( T &v ) __TBB_override { - return try_get(v); - } - - //! Releases the reserved item - bool try_release() __TBB_override { return true; } - - //! Consumes the reserved item - bool try_consume() __TBB_override { return true; } -#endif - - bool is_valid() { - spin_mutex::scoped_lock l( my_mutex ); - return my_buffer_is_valid; - } - - void clear() { - spin_mutex::scoped_lock l( my_mutex ); - my_buffer_is_valid = false; + my_built_predecessors.receiver_extract(*this); + my_successors.built_successors().sender_extract(*this); } +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: - template< typename R, typename B > friend class run_and_put_task; template friend class internal::broadcast_cache; template friend class internal::round_robin_cache; - task * try_put_task( const input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return try_put_task_impl(v); - } - - task * try_put_task_impl(const input_type &v) { - my_buffer = v; - my_buffer_is_valid = true; - task * rtask = my_successors.try_put_task(v); - if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; - return rtask; + //! build a task to run the successor if possible. Default is old behavior. + task *try_put_task(const T& t) __TBB_override { + task *new_task = my_successors.try_put_task(t); + if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; + return new_task; } graph& graph_reference() __TBB_override { return my_graph; } -#if TBB_PREVIEW_RESERVABLE_OVERWRITE_NODE - //! Breaks an infinite loop between the node reservation and register_successor call - struct register_predecessor_task : public task { - register_predecessor_task(sender& owner, receiver& succ) : - o(owner), s(succ) {}; - - tbb::task* execute() __TBB_override { - if (!s.register_predecessor(o)) { - o.register_successor(s); - } - return NULL; - } - - sender& o; - receiver& s; - }; -#endif - - spin_mutex my_mutex; - internal::broadcast_cache< input_type, null_rw_mutex > my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; -#endif - input_type my_buffer; - bool my_buffer_is_valid; void reset_receiver(reset_flags /*f*/) __TBB_override {} - void reset_node( reset_flags f) __TBB_override { - my_buffer_is_valid = false; - if (f&rf_clear_edges) { + void reset_node(reset_flags f) __TBB_override { + if (f&rf_clear_edges) { my_successors.clear(); - } - } -}; // overwrite_node - -template< typename T > -class write_once_node : public overwrite_node { -public: - typedef T input_type; - typedef T output_type; - typedef overwrite_node base_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - explicit write_once_node(graph& g) : base_type(g) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Copy constructor: call base class copy constructor - write_once_node( const write_once_node& src ) : base_type(src) { - tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + my_built_predecessors.clear(); #endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T &v ) __TBB_override { - spin_mutex::scoped_lock l( this->my_mutex ); - return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); + } + __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); } -}; +}; // broadcast_node -//! Forwards messages of type T to all successors -template -class broadcast_node : public graph_node, public receiver, public sender { +//! Forwards messages in arbitrary order +template > +class buffer_node : public graph_node, public internal::reservable_item_buffer, public receiver, public sender { public: typedef T input_type; typedef T output_type; typedef typename receiver::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + typedef buffer_node class_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename sender::successor_list_type successor_list_type; #endif -private: - internal::broadcast_cache my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - internal::edge_container my_built_predecessors; - spin_mutex pred_mutex; // serialize accesses on edge_container -#endif -public: - - explicit broadcast_node(graph& g) : graph_node(g) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - // Copy constructor - broadcast_node( const broadcast_node& src ) : - graph_node(src.my_graph), receiver(), sender() - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Adds a successor - bool register_successor( successor_type &r ) __TBB_override { - my_successors.register_successor( r ); - return true; - } - - //! Removes s as a successor - bool remove_successor( successor_type &r ) __TBB_override { - my_successors.remove_successor( r ); - return true; - } - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor(successor_type &r) __TBB_override { - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor(successor_type &r) __TBB_override { - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! build a task to run the successor if possible. Default is old behavior. - task *try_put_task(const T& t) __TBB_override { - task *new_task = my_successors.try_put_task(t); - if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - graph& graph_reference() __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node(reset_flags f) __TBB_override { - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - my_built_predecessors.clear(); -#endif - } - __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); - } -}; // broadcast_node - -//! Forwards messages in arbitrary order -template > -class buffer_node : public graph_node, public internal::reservable_item_buffer, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef buffer_node class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -protected: - typedef size_t size_type; - internal::round_robin_cache< T, null_rw_mutex > my_successors; - -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +protected: + typedef size_t size_type; + internal::round_robin_cache< T, null_rw_mutex > my_successors; + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION internal::edge_container my_built_predecessors; #endif friend class internal::forward_task_bypass< buffer_node< T, A > >; enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, add_blt_pred, del_blt_pred, blt_succ_cnt, blt_pred_cnt, @@ -1799,7 +1572,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer { public: char type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION task * ltask; union { input_type *elem; @@ -1816,7 +1589,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer(&e)) #else , elem(const_cast(&e)) , ltask(NULL) @@ -1852,7 +1625,7 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferstatus, internal::SUCCEEDED); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -1943,7 +1716,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer::built_predecessors_type built_predecessors_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } virtual void internal_add_built_pred(buffer_operation *op) { my_built_predecessors.add_edge(*(op->p)); @@ -1975,7 +1748,7 @@ class buffer_node : public graph_node, public internal::reservable_item_bufferstatus, internal::SUCCEEDED); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ private: void order() {} @@ -2099,7 +1872,7 @@ class buffer_node : public graph_node, public internal::reservable_item_buffer { } }; // priority_queue_node +} // interfaceX + +namespace interface11 { + +using namespace interface10; +namespace internal = interface10::internal; + //! Forwards messages only if the threshold has not been reached /** This node forwards items until its threshold is reached. It contains no buffering. If the downstream node rejects, the message is dropped. */ -template< typename T > +template< typename T, typename DecrementType=continue_msg > class limiter_node : public graph_node, public receiver< T >, public sender< T > { public: typedef T input_type; typedef T output_type; typedef typename receiver::predecessor_type predecessor_type; typedef typename sender::successor_type successor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename sender::built_successors_type built_successors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; @@ -2647,12 +2427,12 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; spin_mutex my_mutex; internal::broadcast_cache< T > my_successors; - int init_decrement_predecessors; + __TBB_DEPRECATED_LIMITER_EXPR( int init_decrement_predecessors; ) - friend class internal::forward_task_bypass< limiter_node >; + friend class internal::forward_task_bypass< limiter_node >; // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node >; + friend class internal::decrementer< limiter_node, DecrementType >; bool check_conditions() { // always called under lock return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); @@ -2685,7 +2465,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( check_conditions() ) { if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *rtask); } } @@ -2703,7 +2483,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( check_conditions() ) { if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); __TBB_ASSERT(!rval, "Have two tasks to handle"); return rtask; } @@ -2717,45 +2497,59 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return; } - task * decrement_counter() { + task* decrement_counter( long long delta ) { { spin_mutex::scoped_lock lock(my_mutex); - if(my_count) --my_count; + if( delta > 0 && size_t(delta) > my_count ) + my_count = 0; + else if( delta < 0 && size_t(delta) > my_threshold - my_count ) + my_count = my_threshold; + else + my_count -= size_t(delta); // absolute value of delta is sufficiently small } return forward_task(); } -public: - //! The internal receiver< continue_msg > that decrements the count - internal::decrementer< limiter_node > decrement; - - //! Constructor - limiter_node(graph &g, size_t threshold, int num_decrement_predecessors=0) : - graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors) - { + void initialize() { my_predecessors.set_owner(this); my_successors.set_owner(this); decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + tbb::internal::fgt_node( + tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(&decrement), + static_cast *>(this) + ); + } +public: + //! The internal receiver< DecrementType > that decrements the count + internal::decrementer< limiter_node, DecrementType > decrement; + +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), + "Deprecated interface of the limiter node can be used only in conjunction " + "with continue_msg as the type of DecrementType template parameter." ); +#endif // Check for incompatible interface + + //! Constructor + limiter_node(graph &g, + __TBB_DEPRECATED_LIMITER_ARG2(size_t threshold, int num_decrement_predecessors=0)) + : graph_node(g), my_threshold(threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(num_decrement_predecessors), + decrement(num_decrement_predecessors)) { + initialize(); } //! Copy constructor limiter_node( const limiter_node& src ) : graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), my_tries(0), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + my_threshold(src.my_threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(src.init_decrement_predecessors), + decrement(src.init_decrement_predecessors)) { + initialize(); } #if TBB_PREVIEW_FLOW_GRAPH_TRACE @@ -2773,7 +2567,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { if ( internal::is_graph_active(this->my_graph) ) { task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); + internal::forward_task_bypass < limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *task); } } @@ -2788,7 +2582,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } @@ -2826,7 +2620,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > my_predecessors.built_predecessors().receiver_extract(*this); decrement.built_predecessors().receiver_extract(decrement); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Adds src to the list of cached predecessors. bool register_predecessor( predecessor_type &src ) __TBB_override { @@ -2834,7 +2628,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > my_predecessors.add( src ); if ( my_count + my_tries < my_threshold && !my_successors.empty() && internal::is_graph_active(this->my_graph) ) { task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); + internal::forward_task_bypass < limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *task); } return true; @@ -2868,7 +2662,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > --my_tries; if (check_conditions() && internal::is_graph_active(this->my_graph)) { rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); } } else { @@ -2879,9 +2673,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return rtask; } - graph& graph_reference() __TBB_override { - return my_graph; - } + graph& graph_reference() __TBB_override { return my_graph; } void reset_receiver(reset_flags /*f*/) __TBB_override { __TBB_ASSERT(false,NULL); // should never be called @@ -2900,6 +2692,9 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > decrement.reset_receiver(f); } }; // limiter_node +} // namespace interfaceX + +namespace interface10 { #include "internal/_flow_graph_join_impl.h" @@ -3084,7 +2879,7 @@ class indexer_node : public internal::unfolded_indexer_node > { } #if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) { + void set_name( const char *name ) __TBB_override { tbb::internal::fgt_node_desc( this, name ); } #endif @@ -3109,7 +2904,7 @@ class indexer_node : public internal::unfolded_indexer_node : public internal::unfolded_indexer_node inline void internal_make_edge( sender &p, receiver &s ) { #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION s.internal_add_built_predecessor(p); p.internal_add_built_successor(s); #endif @@ -3399,7 +3194,7 @@ template< typename T > inline void internal_remove_edge( sender &p, receiver &s ) { #endif p.remove_successor( s ); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // TODO: should we try to remove p from the predecessor list of s, in case the edge is reversed? p.internal_delete_built_successor(s); s.internal_delete_built_predecessor(p); @@ -3454,7 +3249,7 @@ inline void remove_edge( sender& output, V& input) { } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template template< typename S > void internal::edge_container::sender_extract( S &s ) { @@ -3472,7 +3267,7 @@ void internal::edge_container::receiver_extract( R &r ) { remove_edge(**i, r); } } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ //! Returns a copy of the body from a function or continue node template< typename Body, typename Node > @@ -3493,9 +3288,6 @@ class composite_node , tbb::flow::tuple&... > output_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif std::unique_ptr my_input_ports; std::unique_ptr my_output_ports; @@ -3507,38 +3299,32 @@ class composite_node , tbb::flow::tuplemy_graph ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } -#endif +#else composite_node( graph &g ) : graph_node(g) { tbb::internal::fgt_multiinput_multioutput_node( tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); } +#endif - template - void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); - my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); + template + void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { + __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); + __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); + my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); + my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); -#endif - } + tbb::internal::fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + tbb::internal::fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template< typename... NodeTypes > void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template< typename... NodeTypes > void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) { } - template void add_visible_nodes(Nodes&...) { } -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3556,7 +3342,7 @@ class composite_node , tbb::flow::tuple, tbb::flow::tuple<> > : pu typedef tbb::flow::tuple< receiver&... > input_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif std::unique_ptr my_input_ports; static const size_t NUM_INPUTS = sizeof...(InputTypes); @@ -3581,12 +3364,14 @@ class composite_node , tbb::flow::tuple<> > : pu public: #if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); + composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } #else - composite_node( graph &g) : graph_node(g) {} + composite_node( graph &g ) : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + } #endif template @@ -3595,21 +3380,14 @@ class composite_node , tbb::flow::tuple<> > : pu my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, std::forward(input_ports_tuple)); -#endif + tbb::internal::fgt_internal_input_alias_helper::alias_port( this, std::forward(input_ports_tuple)); } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template< typename... NodeTypes > void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template< typename... NodeTypes > void add_nodes( const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3622,7 +3400,7 @@ class composite_node , tbb::flow::tuple<> > : pu return *my_input_ports; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); } @@ -3637,9 +3415,6 @@ class composite_node , tbb::flow::tuple > : p typedef tbb::flow::tuple< sender&... > output_ports_type; private: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - const char *my_type_name; -#endif std::unique_ptr my_output_ports; static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); @@ -3648,12 +3423,14 @@ class composite_node , tbb::flow::tuple > : p public: #if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g), my_type_name(type_name) { - tbb::internal::itt_make_task_group( tbb::internal::ITT_DOMAIN_FLOW, this, tbb::internal::FLOW_NODE, &g, tbb::internal::FLOW_GRAPH, tbb::internal::FLOW_COMPOSITE_NODE ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, my_type_name ); + composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); } #else - composite_node( graph &g) : graph_node(g) {} + composite_node( graph &g ) : graph_node(g) { + tbb::internal::fgt_composite( this, &g ); + } #endif template @@ -3662,21 +3439,14 @@ class composite_node , tbb::flow::tuple > : p my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, std::forward(output_ports_tuple)); -#endif + tbb::internal::fgt_internal_output_alias_helper::alias_port( this, std::forward(output_ports_tuple)); } -#if TBB_PREVIEW_FLOW_GRAPH_TRACE template void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } template void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } -#else - template void add_nodes(Nodes&...) {} - template void add_visible_nodes(Nodes&...) {} -#endif #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { @@ -3689,7 +3459,7 @@ class composite_node , tbb::flow::tuple > : p return *my_output_ports; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); } @@ -3721,7 +3491,7 @@ class async_body: public async_body_base { typedef async_body_base base_type; typedef Gateway gateway_type; - async_body(const Body &body, gateway_type *gateway) + async_body(const Body &body, gateway_type *gateway) : base_type(gateway), my_body(body) { } void operator()( const Input &v, Ports & ) { @@ -3737,10 +3507,12 @@ class async_body: public async_body_base { } //! Implements async node -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > +template < typename Input, typename Output, + typename Policy = queueing_lightweight, + typename Allocator=cache_aligned_allocator > class async_node : public multifunction_node< Input, tuple< Output >, Policy, Allocator >, public sender< Output > { typedef multifunction_node< Input, tuple< Output >, Policy, Allocator > base_type; - typedef typename internal::multifunction_input mfn_input_type; + typedef typename internal::multifunction_input mfn_input_type; public: typedef Input input_type; @@ -3756,6 +3528,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al struct try_put_functor { typedef internal::multifunction_output output_port_type; output_port_type *port; + // TODO: pass value by copy since we do not want to block asynchronous thread. const Output *value; bool result; try_put_functor(output_port_type &p, const Output &v) : port(&p), value(&v), result(false) { } @@ -3792,20 +3565,34 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al //! Implements gateway_type::try_put for an external activity to submit a message to FG bool try_put_impl(const Output &i) { internal::multifunction_output &port_0 = internal::output_port<0>(*this); + internal::broadcast_cache& port_successors = port_0.successors(); tbb::internal::fgt_async_try_put_begin(this, &port_0); - try_put_functor tpf(port_0, i); - internal::execute_in_graph_arena(this->my_graph, tpf); + task_list tasks; + bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); + __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), + "Return status is inconsistent with the method operation." ); + + while( !tasks.empty() ) { + internal::enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); + } tbb::internal::fgt_async_try_put_end(this, &port_0); - return tpf.result; + return is_at_least_one_put_successful; } public: template - async_node( graph &g, size_t concurrency, Body body ) : - base_type( g, concurrency, internal::async_body(body, &my_gateway) ), my_gateway(self()) { - tbb::internal::fgt_multioutput_node_with_body<1>( tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); + async_node( + graph &g, size_t concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) + ) : base_type( + g, concurrency, + internal::async_body + (body, &my_gateway) __TBB_FLOW_GRAPH_PRIORITY_ARG0(priority) ), my_gateway(self()) { + tbb::internal::fgt_multioutput_node_with_body<1>( + tbb::internal::FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); } async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { @@ -3823,7 +3610,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); + tbb::internal::fgt_multioutput_node_desc( this, name ); } #endif @@ -3848,7 +3635,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al return ab.get_body(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! interface to record edges for traversal & deletion typedef typename internal::edge_container built_successors_type; typedef typename built_successors_type::edge_list_type successor_list_type; @@ -3871,7 +3658,7 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al size_t successor_count() __TBB_override { return internal::output_port<0>(*this).successor_count(); } -#endif +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: @@ -3884,6 +3671,257 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al #include "internal/_flow_graph_streaming_node.h" #endif // __TBB_PREVIEW_STREAMING_NODE +} // interfaceX + + +namespace interface10a { + +using namespace interface10; +namespace internal = interface10::internal; + +template< typename T > +class overwrite_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename receiver::built_predecessors_type built_predecessors_type; + typedef typename sender::built_successors_type built_successors_type; + typedef typename receiver::predecessor_list_type predecessor_list_type; + typedef typename sender::successor_list_type successor_list_type; +#endif + + explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { + my_successors.set_owner( this ); + tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + + //! Copy constructor; doesn't take anything from src; default won't work + overwrite_node( const overwrite_node& src ) : + graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) + { + my_successors.set_owner( this ); + tbb::internal::fgt_node( tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + + ~overwrite_node() {} + +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + void set_name( const char *name ) __TBB_override { + tbb::internal::fgt_node_desc( this, name ); + } +#endif + + bool register_successor( successor_type &s ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid && internal::is_graph_active( my_graph )) { + // We have a valid value that must be forwarded immediately. + bool ret = s.try_put( my_buffer ); + if ( ret ) { + // We add the successor that accepted our put + my_successors.register_successor( s ); + } else { + // In case of reservation a race between the moment of reservation and register_successor can appear, + // because failed reserve does not mean that register_successor is not ready to put a message immediately. + // We have some sort of infinite loop: reserving node tries to set pull state for the edge, + // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. + task *rtask = new ( task::allocate_additional_child_of( *( my_graph.root_task() ) ) ) + register_predecessor_task( *this, s ); + internal::spawn_in_graph_arena( my_graph, *rtask ); + } + } else { + // No valid value yet, just add as successor + my_successors.register_successor( s ); + } + return true; + } + + bool remove_successor( successor_type &s ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.remove_successor(s); + return true; + } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } + + void internal_add_built_successor( successor_type &s) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.internal_add_built_successor(s); + } + + void internal_delete_built_successor( successor_type &s) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.internal_delete_built_successor(s); + } + + size_t successor_count() __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return my_successors.successor_count(); + } + + void copy_successors(successor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.copy_successors(v); + } + + void internal_add_built_predecessor( predecessor_type &p) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.add_edge(p); + } + + void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.delete_edge(p); + } + + size_t predecessor_count() __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return my_built_predecessors.edge_count(); + } + + void copy_predecessors( predecessor_list_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + my_built_predecessors.copy_edges(v); + } + + void extract() __TBB_override { + my_buffer_is_valid = false; + built_successors().sender_extract(*this); + built_predecessors().receiver_extract(*this); + } + +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + + bool try_get( input_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + if ( my_buffer_is_valid ) { + v = my_buffer; + return true; + } + return false; + } + + //! Reserves an item + bool try_reserve( T &v ) __TBB_override { + return try_get(v); + } + + //! Releases the reserved item + bool try_release() __TBB_override { return true; } + + //! Consumes the reserved item + bool try_consume() __TBB_override { return true; } + + bool is_valid() { + spin_mutex::scoped_lock l( my_mutex ); + return my_buffer_is_valid; + } + + void clear() { + spin_mutex::scoped_lock l( my_mutex ); + my_buffer_is_valid = false; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class internal::broadcast_cache; + template friend class internal::round_robin_cache; + task * try_put_task( const input_type &v ) __TBB_override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v); + } + + task * try_put_task_impl(const input_type &v) { + my_buffer = v; + my_buffer_is_valid = true; + task * rtask = my_successors.try_put_task(v); + if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; + return rtask; + } + + graph& graph_reference() __TBB_override { + return my_graph; + } + + //! Breaks an infinite loop between the node reservation and register_successor call + struct register_predecessor_task : public graph_task { + + register_predecessor_task(predecessor_type& owner, successor_type& succ) : + o(owner), s(succ) {}; + + tbb::task* execute() __TBB_override { + if (!s.register_predecessor(o)) { + o.register_successor(s); + } + return NULL; + } + + predecessor_type& o; + successor_type& s; + }; + + spin_mutex my_mutex; + internal::broadcast_cache< input_type, null_rw_mutex > my_successors; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + internal::edge_container my_built_predecessors; +#endif + input_type my_buffer; + bool my_buffer_is_valid; + void reset_receiver(reset_flags /*f*/) __TBB_override {} + + void reset_node( reset_flags f) __TBB_override { + my_buffer_is_valid = false; + if (f&rf_clear_edges) { + my_successors.clear(); + } + } +}; // overwrite_node + +template< typename T > +class write_once_node : public overwrite_node { +public: + typedef T input_type; + typedef T output_type; + typedef overwrite_node base_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + explicit write_once_node(graph& g) : base_type(g) { + tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + //! Copy constructor: call base class copy constructor + write_once_node( const write_once_node& src ) : base_type(src) { + tbb::internal::fgt_node( tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + void set_name( const char *name ) __TBB_override { + tbb::internal::fgt_node_desc( this, name ); + } +#endif + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class internal::broadcast_cache; + template friend class internal::round_robin_cache; + task *try_put_task( const T &v ) __TBB_override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); + } +}; } // interfaceX using interface10::reset_flags; @@ -3905,14 +3943,14 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al using interface10::internal::cast_to; using interface10::internal::is_a; using interface10::continue_node; - using interface10::overwrite_node; - using interface10::write_once_node; + using interface10a::overwrite_node; + using interface10a::write_once_node; using interface10::broadcast_node; using interface10::buffer_node; using interface10::queue_node; using interface10::sequencer_node; using interface10::priority_queue_node; - using interface10::limiter_node; + using interface11::limiter_node; using namespace interface10::internal::graph_policy_namespace; using interface10::join_node; using interface10::input_port; @@ -3931,6 +3969,11 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al using interface10::port_ref; using interface10::streaming_node; #endif // __TBB_PREVIEW_STREAMING_NODE +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + using internal::node_priority_t; + using internal::no_priority; +#endif + } // flow } // tbb diff --git a/src/tbb/include/tbb/flow_graph_abstractions.h b/src/tbb/include/tbb/flow_graph_abstractions.h index 46971e03..e690f0d3 100644 --- a/src/tbb/include/tbb/flow_graph_abstractions.h +++ b/src/tbb/include/tbb/flow_graph_abstractions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_abstractions_H diff --git a/src/tbb/include/tbb/flow_graph_opencl_node.h b/src/tbb/include/tbb/flow_graph_opencl_node.h index e28abe78..2c99fb33 100644 --- a/src/tbb/include/tbb/flow_graph_opencl_node.h +++ b/src/tbb/include/tbb/flow_graph_opencl_node.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_opencl_node_H @@ -121,19 +117,19 @@ class opencl_device { opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ), my_cl_command_queue( NULL ) {} std::string platform_profile() const { - return platform_info( platform(), CL_PLATFORM_PROFILE ); + return platform_info( platform_id(), CL_PLATFORM_PROFILE ); } std::string platform_version() const { - return platform_info( platform(), CL_PLATFORM_VERSION ); + return platform_info( platform_id(), CL_PLATFORM_VERSION ); } std::string platform_name() const { - return platform_info( platform(), CL_PLATFORM_NAME ); + return platform_info( platform_id(), CL_PLATFORM_NAME ); } std::string platform_vendor() const { - return platform_info( platform(), CL_PLATFORM_VENDOR ); + return platform_info( platform_id(), CL_PLATFORM_VENDOR ); } std::string platform_extensions() const { - return platform_info( platform(), CL_PLATFORM_EXTENSIONS ); + return platform_info( platform_id(), CL_PLATFORM_EXTENSIONS ); } template @@ -229,12 +225,12 @@ class opencl_device { my_cl_command_queue = cmd_queue; } -private: - - cl_platform_id platform() const { + cl_platform_id platform_id() const { return device_info( my_cl_device_id, CL_DEVICE_PLATFORM ); } +private: + device_id_type my_device_id; cl_device_id my_cl_device_id; cl_command_queue my_cl_command_queue; @@ -457,7 +453,7 @@ class opencl_async_msg : public async_msg { operator const T&() const { return data(); } protected: - // Overridden in this derived class to inform that + // Overridden in this derived class to inform that // async calculation chain is over void finalize() const __TBB_override { receive_if_memory_object(*this); @@ -530,10 +526,10 @@ class opencl_memory { opencl_async_msg receive(const cl_event *e) { opencl_async_msg d; - if (e) { + if (e) { d = opencl_async_msg(my_host_ptr, *e); - } else { - d = opencl_async_msg(my_host_ptr); + } else { + d = opencl_async_msg(my_host_ptr); } // Concurrent receives are prohibited so we do not worry about synchronization. @@ -1089,9 +1085,9 @@ class opencl_factory { } enforce_cl_retcode(my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list"); - cl_platform_id platform_id = my_devices.begin()->platform(); + cl_platform_id platform_id = my_devices.begin()->platform_id(); for (opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it) - enforce_cl_retcode(it->platform() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform"); + enforce_cl_retcode(it->platform_id() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform"); std::vector cl_device_ids; for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { @@ -1128,7 +1124,7 @@ class opencl_factory { // Suppress "declared deprecated" warning for the next line. #if __TBB_GCC_WARNING_SUPPRESSION_PRESENT #pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif #if _MSC_VER || __INTEL_COMPILER #pragma warning( push ) @@ -1181,7 +1177,12 @@ struct default_device_selector { struct default_device_filter { opencl_device_list operator()(const opencl_device_list &devices) { opencl_device_list dl; - dl.add(*devices.begin()); + cl_platform_id platform_id = devices.begin()->platform_id(); + for (opencl_device_list::const_iterator it = devices.cbegin(); it != devices.cend(); ++it) { + if (it->platform_id() == platform_id) { + dl.add(*it); + } + } return dl; } }; diff --git a/src/tbb/include/tbb/global_control.h b/src/tbb/include/tbb/global_control.h index fe742020..505720cb 100644 --- a/src/tbb/include/tbb/global_control.h +++ b/src/tbb/include/tbb/global_control.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,19 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_global_control_H #define __TBB_global_control_H -#if !TBB_PREVIEW_GLOBAL_CONTROL && !__TBB_BUILD -#error Set TBB_PREVIEW_GLOBAL_CONTROL before including global_control.h -#endif - #include "tbb_stddef.h" namespace tbb { @@ -41,8 +33,8 @@ class global_control { global_control(parameter p, size_t value) : my_value(value), my_next(NULL), my_param(p) { __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size if (p==thread_stack_size) return; #elif __TBB_x86_64 && (_WIN32 || _WIN64) @@ -56,8 +48,8 @@ class global_control { ~global_control() { __TBB_ASSERT(my_param < parameter_max, "Invalid parameter. Probably the object was corrupted."); -#if __TBB_WIN8UI_SUPPORT - // For Windows Store* apps it's impossible to set stack size +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size if (my_param==thread_stack_size) return; #endif diff --git a/src/tbb/include/tbb/index.html b/src/tbb/include/tbb/index.html index 2eddd450..7bbb672d 100644 --- a/src/tbb/include/tbb/index.html +++ b/src/tbb/include/tbb/index.html @@ -19,7 +19,7 @@

Directories

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/include/tbb/internal/_aggregator_impl.h b/src/tbb/include/tbb/internal/_aggregator_impl.h index 40bbd491..cece90f3 100644 --- a/src/tbb/include/tbb/internal/_aggregator_impl.h +++ b/src/tbb/include/tbb/internal/_aggregator_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__aggregator_impl_H diff --git a/src/tbb/include/tbb/internal/_allocator_traits.h b/src/tbb/include/tbb/internal/_allocator_traits.h new file mode 100644 index 00000000..272077a6 --- /dev/null +++ b/src/tbb/include/tbb/internal/_allocator_traits.h @@ -0,0 +1,156 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_allocator_traits_H +#define __TBB_allocator_traits_H + +#include "../tbb_stddef.h" // true/false_type + +#if __TBB_ALLOCATOR_TRAITS_PRESENT +#include // for allocator_traits +#endif + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include // for std::move +#endif + +// For allocator_swap helper +#include __TBB_STD_SWAP_HEADER + +namespace tbb { +namespace internal { + +//! Internal implementation of allocator traits, propagate_on_* use internal boolean_constant. +//! In order to avoid code duplication, check what implementation of boolean constant will likely be passed. +#if __TBB_ALLOCATOR_TRAITS_PRESENT +typedef std::true_type traits_true_type; +typedef std::false_type traits_false_type; +#else +typedef tbb::internal::true_type traits_true_type; +typedef tbb::internal::false_type traits_false_type; +#endif + +//! Copy assignment implementation for allocator if propagate_on_container_copy_assignment == true_type +//! Noop if pocca == false_type +template +inline void allocator_copy_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + my_allocator = other_allocator; +} +template +inline void allocator_copy_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO COPY */} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +//! Move assignment implementation for allocator if propagate_on_container_move_assignment == true_type. +//! Noop if pocma == false_type. +template +inline void allocator_move_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + my_allocator = std::move(other_allocator); +} +template +inline void allocator_move_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO MOVE */ } +#endif + +//! Swap implementation for allocators if propagate_on_container_swap == true_type. +//! Noop if pocs == false_type. +template +inline void allocator_swap(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { + using std::swap; + swap(my_allocator, other_allocator); +} +template +inline void allocator_swap(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO SWAP */ } + +#if __TBB_ALLOCATOR_TRAITS_PRESENT +using std::allocator_traits; +#else +//! Internal allocator_traits implementation, which relies on C++03 standard +//! [20.1.5] allocator requirements +template +struct allocator_traits { + // C++03 allocator doesn't have to be assignable or swappable, therefore + // define these traits as false_type to do not require additional operations + // that are not supposed to be in. + typedef tbb::internal::false_type propagate_on_container_move_assignment; + typedef tbb::internal::false_type propagate_on_container_copy_assignment; + typedef tbb::internal::false_type propagate_on_container_swap; + + typedef Alloc allocator_type; + typedef typename allocator_type::value_type value_type; + + typedef typename allocator_type::pointer pointer; + typedef typename allocator_type::const_pointer const_pointer; + typedef typename allocator_type::difference_type difference_type; + typedef typename allocator_type::size_type size_type; + + template struct rebind_alloc { + typedef typename Alloc::template rebind::other other; + }; + + static pointer allocate(Alloc& a, size_type n) { + return a.allocate(n); + } + + static void deallocate(Alloc& a, pointer p, size_type n) { + a.deallocate(p, n); + } + + template + static void construct(Alloc&, PT* p) { + ::new (static_cast(p)) PT(); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1)); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, __TBB_FORWARDING_REF(T2) t2) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2)); + } + + template + static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, + __TBB_FORWARDING_REF(T2) t2, __TBB_FORWARDING_REF(T3) t3) { + ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2), + tbb::internal::forward(t3)); + } + + template + static void destroy(Alloc&, T* p) { + p->~T(); + tbb::internal::suppress_unused_warning(p); + } + + static Alloc select_on_container_copy_construction(const Alloc& a) { return a; } +}; +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + +//! C++03/C++11 compliant rebind helper, even if no std::allocator_traits available +//! or rebind is not defined for allocator type +template +struct allocator_rebind { +#if __TBB_ALLOCATOR_TRAITS_PRESENT + typedef typename allocator_traits::template rebind_alloc type; +#else + typedef typename allocator_traits::template rebind_alloc::other type; +#endif +}; + +}} // namespace tbb::internal + +#endif // __TBB_allocator_traits_H + diff --git a/src/tbb/include/tbb/internal/_concurrent_queue_impl.h b/src/tbb/include/tbb/internal/_concurrent_queue_impl.h index 29e27426..c1aafc9e 100644 --- a/src/tbb/include/tbb/internal/_concurrent_queue_impl.h +++ b/src/tbb/include/tbb/internal/_concurrent_queue_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__concurrent_queue_impl_H @@ -514,7 +510,7 @@ concurrent_queue_base_v3::concurrent_queue_base_v3() { __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset(my_rep,0,sizeof(concurrent_queue_rep)); + memset(static_cast(my_rep),0,sizeof(concurrent_queue_rep)); my_rep->item_size = item_size; my_rep->items_per_page = item_size<= 8 ? 32 : item_size<= 16 ? 16 : @@ -889,7 +885,7 @@ class concurrent_queue_base_v3: no_copy { //! Get size of queue ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - //! Check if the queue is emtpy + //! Check if the queue is empty bool __TBB_EXPORTED_METHOD internal_empty() const; //! Set the queue capacity diff --git a/src/tbb/include/tbb/internal/_concurrent_skip_list_impl.h b/src/tbb/include/tbb/internal/_concurrent_skip_list_impl.h new file mode 100644 index 00000000..685c45ae --- /dev/null +++ b/src/tbb/include/tbb/internal/_concurrent_skip_list_impl.h @@ -0,0 +1,1043 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_skip_list_H +#define __TBB_concurrent_skip_list_H + +#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "../tbb_config.h" +#include "../tbb_stddef.h" +#include "../tbb_allocator.h" +#include "../spin_mutex.h" +#include "../tbb_exception.h" +#include "../enumerable_thread_specific.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_node_handle_impl.h" +#include // Need std::pair +#include +#include +#include // Need std::allocator_traits +#include +#include +#include +#include +#include +#include +#include +#include + +#if _MSC_VER +#pragma warning(disable: 4189) // warning 4189 -- local variable is initialized but not referenced +#pragma warning(disable: 4127) // warning 4127 -- while (true) has a constant expression in it +#endif + +namespace tbb { +namespace interface10 { +namespace internal { + +template +class skip_list_node { + +public: + using value_type = Value; + using size_type = std::size_t; + using reference = value_type & ; + using const_reference = const value_type & ; + using pointer = value_type * ; + using const_pointer = const value_type *; + using node_pointer = skip_list_node * ; + using atomic_node_pointer = std::atomic; + + using mutex_type = Mutex; + using lock_type = std::unique_lock; + + skip_list_node(size_type levels) : my_height(levels), my_fullyLinked(false) { + for (size_type lev = 0; lev < my_height; ++lev) + new(&my_next(lev)) atomic_node_pointer(nullptr); + __TBB_ASSERT(height() == levels, "Wrong node height"); + } + + ~skip_list_node() { + for(size_type lev = 0; lev < my_height; ++lev) + my_next(lev).~atomic(); + } + + skip_list_node(const skip_list_node&) = delete; + + skip_list_node(skip_list_node&&) = delete; + + skip_list_node& operator=(const skip_list_node&) = delete; + + pointer storage() { + return reinterpret_cast(&my_val); + } + + reference value() { + return *storage(); + } + + node_pointer next(size_type level) const { + __TBB_ASSERT(level < height(), "Cannot get next on the level greater than height"); + return my_next(level).load(std::memory_order_acquire); + } + + void set_next(size_type level, node_pointer next) { + __TBB_ASSERT(level < height(), "Cannot set next on the level greater than height"); + + my_next(level).store(next, std::memory_order_release); + } + + /** @return number of layers */ + size_type height() const { + return my_height; + } + + bool fully_linked() const { + return my_fullyLinked.load(std::memory_order_acquire); + } + + void mark_linked() { + my_fullyLinked.store(true, std::memory_order_release); + } + + lock_type acquire() { + return lock_type(my_mutex); + } + +private: + using aligned_storage_type = typename std::aligned_storage::type; + + atomic_node_pointer& my_next(size_type level) { + atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + const atomic_node_pointer& my_next(size_type level) const { + const atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + mutex_type my_mutex; + aligned_storage_type my_val; + size_type my_height; + std::atomic_bool my_fullyLinked; +}; + +template +class skip_list_iterator { + using node_type = NodeType; + using node_ptr = node_type*; +public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename node_type::value_type; + using difference_type = std::ptrdiff_t; + using pointer = typename std::conditional::type; + using reference = typename std::conditional::type; + + skip_list_iterator() : my_node_ptr(nullptr) {} + + // TODO: the code above does not compile in VS2015 (seems like a bug) - consider enabling it for all other platforms + // template ::type> + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + template ::type> + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + reference operator*() const { return *(my_node_ptr->storage()); } + pointer operator->() const { return &**this; } + + skip_list_iterator& operator++() { + __TBB_ASSERT(my_node_ptr != nullptr, NULL); + my_node_ptr = my_node_ptr->next(0); + return *this; + } + + skip_list_iterator operator++(int) { + skip_list_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + skip_list_iterator(node_type* n) : my_node_ptr(n) {} + + node_ptr my_node_ptr; + + template + friend class concurrent_skip_list; + + friend class skip_list_iterator; + + friend class const_range; + friend class range; + + template + friend bool operator==(const skip_list_iterator&, const skip_list_iterator&); + + template + friend bool operator!=(const skip_list_iterator&, const skip_list_iterator&); +}; + +template +bool operator==(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr == rhs.my_node_ptr; +} + +template +bool operator!=(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr != rhs.my_node_ptr; +} + +template +class concurrent_skip_list { +protected: + using traits_type = Traits; + using allocator_type = typename traits_type::allocator_type; + using allocator_traits_type = std::allocator_traits; + using key_compare = typename traits_type::compare_type; + using value_compare = typename traits_type::value_compare; + using key_type = typename traits_type::key_type; + using value_type = typename traits_type::value_type; + using node_type = typename traits_type::node_type; + using list_node_type = skip_list_node; + + using iterator = skip_list_iterator; + using const_iterator = skip_list_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using random_level_generator_type = typename traits_type::random_level_generator_type; + using node_allocator_type = typename std::allocator_traits::template rebind_alloc; + using node_allocator_traits = typename std::allocator_traits::template rebind_traits; + using node_ptr = list_node_type*; + + static constexpr size_type MAX_LEVEL = traits_type::MAX_LEVEL; + + using array_type = std::array; + using lock_array = std::array; + +public: + static bool const allow_multimapping = traits_type::allow_multimapping; + + /** + * Default constructor. Construct empty skip list. + */ + concurrent_skip_list() : my_size(0) { + create_dummy_head(); + } + + explicit concurrent_skip_list(const key_compare& comp, const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + } + + template + concurrent_skip_list(InputIt first, InputIt last, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + internal_copy(first, last); + } + + /** Copy constructor */ + concurrent_skip_list(const concurrent_skip_list& other) + : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())), + my_compare(other.my_compare), my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(const concurrent_skip_list& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(concurrent_skip_list&& other) + : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + internal_move(std::move(other)); + } + + concurrent_skip_list(concurrent_skip_list&& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + if (alloc == other.get_allocator()) { + internal_move(std::move(other)); + } else { + my_size = 0; + create_dummy_head(); + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + ~concurrent_skip_list() { + clear(); + delete_dummy_head(); + } + + concurrent_skip_list& operator=(const concurrent_skip_list& other) { + if (this != &other) { + using pocca_type = typename node_allocator_traits::propagate_on_container_copy_assignment; + clear(); + tbb::internal::allocator_copy_assignment(my_node_allocator, other.my_node_allocator, pocca_type()); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_copy(other); + } + return *this; + } + + concurrent_skip_list& operator=(concurrent_skip_list&& other) { + if (this != &other) { + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + clear(); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_move_assign(std::move(other), pocma_type()); + } + return *this; + } + + concurrent_skip_list& operator=(std::initializer_list il) + { + clear(); + insert(il.begin(),il.end()); + return *this; + } + + std::pair insert(const value_type& value) { + return internal_insert(value); + } + + std::pair insert(value_type&& value) { + return internal_insert(std::move(value)); + } + + iterator insert(const_iterator, const_reference value) { + // Ignore hint + return insert(value).first; + } + + iterator insert(const_iterator, value_type&& value) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert(InputIterator first, InputIterator last) { + for (InputIterator it = first; it != last; ++it) + insert(*it); + } + + void insert(std::initializer_list init) { + insert(init.begin(), init.end()); + } + + std::pair insert(node_type&& nh) { + if(!nh.empty()) { + std::pair insert_result = internal_insert_node(nh.my_node); + if(insert_result.second) { + nh.deactivate(); + } + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert(const_iterator, node_type&& nh) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace(Args&&... args) { + return internal_insert(std::forward(args)...); + } + + template + iterator emplace_hint(const_iterator, Args&&... args) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase(iterator pos) { + std::pair extract_result = internal_extract(pos); + if(extract_result.first) { // node was extracted + delete_node(extract_result.first); + return extract_result.second; + } + return end(); + } + + iterator unsafe_erase(const_iterator first, const_iterator last) { + while(first != last) { + first = unsafe_erase(get_iterator(first)); + } + return get_iterator(first); + } + + size_type unsafe_erase(const key_type& key) { + std::pair range = equal_range(key); + size_type sz = std::distance(range.first, range.second); + unsafe_erase(range.first, range.second); + return sz; + } + + node_type unsafe_extract(const_iterator pos) { + std::pair extract_result = internal_extract(pos); + return extract_result.first ? node_type(extract_result.first) : node_type(); + } + + node_type unsafe_extract(const key_type& key) { + return unsafe_extract(find(key)); + } + + iterator lower_bound(const key_type& key) { + return internal_get_bound(key, my_compare); + } + + const_iterator lower_bound(const key_type& key) const { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + iterator lower_bound(const K& key) { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + const_iterator lower_bound(const K& key) const { + return internal_get_bound(key, my_compare); + } + + iterator upper_bound(const key_type& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + const_iterator upper_bound(const key_type& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + iterator upper_bound(const K& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + const_iterator upper_bound(const K& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + iterator find(const key_type& key) { + return internal_find(key); + } + + const_iterator find(const key_type& key) const { + return internal_find(key); + } + + template::value, K>::type> + iterator find(const K& key) { + return internal_find(key); + } + + template::value, K>::type> + const_iterator find(const K& key) const { + return internal_find(key); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template::value, K>::type> + size_type count(const K& key) const { + return internal_count(key); + } + + bool contains(const key_type& key) const { + return find(key) != end(); + } + + template::value, K>::type> + bool contains(const K& key) const { + return find(key) != end(); + } + + void clear() noexcept { + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + node_ptr current = dummy_head->next(0); + while (current) { + __TBB_ASSERT(current->height() > 0, NULL); + node_ptr next = current->next(0); + delete_node(current); + current = next; + } + + my_size = 0; + for (size_type i = 0; i < dummy_head->height(); ++i) { + dummy_head->set_next(i, nullptr); + } + } + + iterator begin() { + return iterator(dummy_head->next(0)); + } + + const_iterator begin() const { + return const_iterator(dummy_head->next(0)); + } + + const_iterator cbegin() const { + return const_iterator(dummy_head->next(0)); + } + + iterator end() { + return iterator(nullptr); + } + + const_iterator end() const { + return const_iterator(nullptr); + } + + const_iterator cend() const { + return const_iterator(nullptr); + } + + size_type size() const { + return my_size.load(std::memory_order_relaxed); + } + + size_type max_size() const { + return my_node_allocator.max_size(); + } + + bool empty() const { + return 0 == size(); + } + + allocator_type get_allocator() const { + return my_node_allocator; + } + + void swap(concurrent_skip_list& other) { + using std::swap; + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + tbb::internal::allocator_swap(my_node_allocator, other.my_node_allocator, pocs_type()); + swap(my_compare, other.my_compare); + swap(my_rnd_generator, other.my_rnd_generator); + swap(dummy_head, other.dummy_head); + + size_type tmp = my_size; + my_size.store(other.my_size); + other.my_size.store(tmp); + } + + std::pair equal_range(const key_type& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + std::pair equal_range(const key_type& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + key_compare key_comp() const { return my_compare; } + + value_compare value_comp() const { return traits_type::value_comp(my_compare); } + + class const_range_type : tbb::internal::no_assign { + public: + using size_type = typename concurrent_skip_list::size_type; + using value_type = typename concurrent_skip_list::value_type; + using iterator = typename concurrent_skip_list::const_iterator; + private: + const_iterator my_end; + const_iterator my_begin; + size_type my_level; + + public: + + bool empty() const { + return my_begin.my_node_ptr->next(0) == my_end.my_node_ptr; + } + + bool is_divisible() const { + return my_level != 0 ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr : false; + } + + size_type size() const { return std::distance(my_begin, my_end);} + + const_range_type( const_range_type& r, split) + : my_end(r.my_end) { + my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); + my_level = my_begin.my_node_ptr->height(); + r.my_end = my_begin; + } + + const_range_type( const concurrent_skip_list& l) + : my_end(l.end()), my_begin(l.begin()), my_level(my_begin.my_node_ptr->height() ) {} + + iterator begin() const { return my_begin; } + iterator end() const { return my_end; } + size_t grainsize() const { return 1; } + + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_skip_list::iterator; + + range_type(range_type& r, split) : const_range_type(r, split()) {} + range_type(const concurrent_skip_list& l) : const_range_type(l) {} + + iterator begin() const { + node_ptr node = const_range_type::begin().my_node_ptr; + return iterator(node); + } + + iterator end() const { + node_ptr node = const_range_type::end().my_node_ptr; + return iterator(node); } + }; // class range_type + + range_type range() { return range_type(*this); } + const_range_type range() const { return const_range_type(*this); } + +private: + void internal_move(concurrent_skip_list&& other) { + dummy_head = other.dummy_head; + other.dummy_head = nullptr; + other.create_dummy_head(); + + my_size = other.my_size.load(); + other.my_size = 0; + } + + static const key_type& get_key(node_ptr n) { + __TBB_ASSERT(n, NULL); + return traits_type::get_key(n->value()); + } + + template + iterator internal_find(const K& key) { + iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + const_iterator internal_find(const K& key) const { + const_iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + std::pair range = equal_range(key); + return std::distance(range.first, range.second); + } + return (find(key) == end()) ? size_type(0) : size_type(1); + } + + /** + * Finds position on the @param level using @param cmp + * @param level - on which level search prev node + * @param prev - pointer to the start node to search + * @param key - key to search + * @param cmp - callable object to compare two objects + * (my_compare member is default comparator) + * @returns pointer to the node which is not satisfy the comparison with @param key + */ + template + pointer_type internal_find_position( size_type level, pointer_type& prev, const K& key, + const comparator& cmp) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + pointer_type curr = prev->next(level); + + while (curr && cmp(get_key(curr), key)) { + prev = curr; + __TBB_ASSERT(level < prev->height(), NULL); + curr = prev->next(level); + } + + return curr; + } + + template + void fill_prev_next_arrays(array_type& prev_nodes, array_type& next_nodes, node_ptr prev, const key_type& key, + const comparator& cmp) { + prev_nodes.fill(dummy_head); + next_nodes.fill(nullptr); + + for (size_type h = prev->height(); h > 0; --h) { + node_ptr next = internal_find_position(h - 1, prev, key, cmp); + prev_nodes[h - 1] = prev; + next_nodes[h - 1] = next; + } + } + + template + std::pair internal_insert(Args&&... args) { + node_ptr new_node = create_node(std::forward(args)...); + std::pair insert_result = internal_insert_node(new_node); + if(!insert_result.second) { + delete_node(new_node); + } + return insert_result; + } + + std::pair internal_insert_node(node_ptr new_node) { + array_type prev_nodes; + array_type next_nodes; + __TBB_ASSERT(dummy_head->height() >= new_node->height(), "Wrong height for new node"); + + do { + if (allow_multimapping) { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), + not_greater_compare(my_compare)); + } else { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), my_compare); + } + + node_ptr next = next_nodes[0]; + if (next && !allow_multimapping && !my_compare(get_key(new_node), get_key(next))) { + // TODO: do we really need to wait? + while (!next->fully_linked()) { + // TODO: atomic backoff + } + + return std::pair(iterator(next), false); + } + __TBB_ASSERT(allow_multimapping || !next || my_compare(get_key(new_node), get_key(next)), + "Wrong elements order"); + + } while (!try_insert_node(new_node, prev_nodes, next_nodes)); + + __TBB_ASSERT(new_node, NULL); + return std::pair(iterator(new_node), true); + } + + bool try_insert_node(node_ptr new_node, array_type& prev_nodes, array_type& next_nodes) { + __TBB_ASSERT(dummy_head->height() >= new_node->height(), NULL); + + lock_array locks; + + if (!try_lock_nodes(new_node->height(), prev_nodes, next_nodes, locks)) { + return false; + } + + __TBB_ASSERT(allow_multimapping || + ((prev_nodes[0] == dummy_head || + my_compare(get_key(prev_nodes[0]), get_key(new_node))) && + (next_nodes[0] == nullptr || my_compare(get_key(new_node), get_key(next_nodes[0])))), + "Wrong elements order"); + + for (size_type level = 0; level < new_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(prev_nodes[level]->next(level) == next_nodes[level], NULL); + new_node->set_next(level, next_nodes[level]); + prev_nodes[level]->set_next(level, new_node); + } + new_node->mark_linked(); + + ++my_size; + + return true; + } + + bool try_lock_nodes(size_type height, array_type& prevs, array_type& next_nodes, lock_array& locks) { + for (size_type l = 0; l < height; ++l) { + if (l == 0 || prevs[l] != prevs[l - 1]) + locks[l] = prevs[l]->acquire(); + + node_ptr next = prevs[l]->next(l); + if ( next != next_nodes[l]) return false; + } + + return true; + } + + template + const_iterator internal_get_bound(const K& key, const comparator& cmp) const { + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return const_iterator(next); + } + + template + iterator internal_get_bound(const K& key, const comparator& cmp){ + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return iterator(next); + } + + // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted + std::pair internal_extract(const_iterator it) { + if ( it != end() ) { + key_type key = traits_type::get_key(*it); + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + array_type prev_nodes; + array_type next_nodes; + + fill_prev_next_arrays(prev_nodes, next_nodes, prev, key, my_compare); + + node_ptr erase_node = next_nodes[0]; + node_ptr next_node = erase_node->next(0); + + if (erase_node && !my_compare(key, get_key(erase_node))) { + for(size_type level = 0; level < erase_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(next_nodes[level] == erase_node, NULL); + prev_nodes[level]->set_next(level, erase_node->next(level)); + } + --my_size; + return std::pair(erase_node, next_node); + } + } + return std::pair(nullptr, nullptr); + } + +protected: + template + void internal_merge(SourceType&& source) { + using source_type = typename std::decay::type; + using source_iterator = typename source_type::iterator; + __TBB_STATIC_ASSERT((std::is_same::value), "Incompatible containers cannot be merged"); + + for(source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || !contains(traits_type::get_key(*where))) { + std::pair extract_result = source.internal_extract(where); + + //If the insertion fails - return the node into source + node_type handle(extract_result.first); + __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); + + if (!insert(std::move(handle)).second) { + source.insert(std::move(handle)); + } + handle.deactivate(); + } + } + } + +private: + void internal_copy(const concurrent_skip_list& other) { + internal_copy(other.begin(), other.end()); + } + + template + void internal_copy(Iterator first, Iterator last) { + clear(); + try { + for (auto it = first; it != last; ++it) + insert(*it); + } + catch (...) { + clear(); + delete_dummy_head(); + throw; + } + } + + /** Generate random level */ + size_type random_level() { + return my_rnd_generator(); + } + + static size_type calc_node_size(size_type height) { + return sizeof(list_node_type) + height*sizeof(typename list_node_type::atomic_node_pointer); + } + + /** Creates new node */ + template + node_ptr create_node(Args&&... args) { + size_type levels = random_level(); + + size_type sz = calc_node_size(levels); + + node_ptr node = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + + try { + node_allocator_traits::construct(my_node_allocator, node, levels); + + } + catch(...) { + deallocate_node(node, sz); + throw; + } + + try { + node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); + } + catch (...) { + node_allocator_traits::destroy(my_node_allocator, node); + deallocate_node(node, sz); + throw; + } + + return node; + } + + void create_dummy_head() { + size_type sz = calc_node_size(MAX_LEVEL); + + dummy_head = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + // TODO: investigate linkage fail in debug without this workaround + auto max_level = MAX_LEVEL; + + try { + node_allocator_traits::construct(my_node_allocator, dummy_head, max_level); + } + catch(...) { + deallocate_node(dummy_head, sz); + throw; + } + } + + template + void delete_node(node_ptr node) { + size_type sz = calc_node_size(node->height()); + // Destroy value + if (!is_dummy) node_allocator_traits::destroy(my_node_allocator, node->storage()); + // Destroy node + node_allocator_traits::destroy(my_node_allocator, node); + // Deallocate memory + deallocate_node(node, sz); + } + + void deallocate_node(node_ptr node, size_type sz) { + node_allocator_traits::deallocate(my_node_allocator, reinterpret_cast(node), sz); + } + + void delete_dummy_head() { + delete_node(dummy_head); + } + + static iterator get_iterator(const_iterator it) { + return iterator(it.my_node_ptr); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::true_type) { + delete_dummy_head(); + tbb::internal::allocator_move_assignment(my_node_allocator, other.my_node_allocator, std::true_type()); + internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::false_type) { + if (my_node_allocator == other.my_node_allocator) { + delete_dummy_head(); + internal_move(std::move(other)); + } else { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + struct not_greater_compare { + const key_compare& my_less_compare; + + not_greater_compare(const key_compare& less_compare) : my_less_compare(less_compare) {} + + template + bool operator()(const K1& first, const K2& second) const { + return !my_less_compare(second, first); + } + }; + + node_allocator_type my_node_allocator; + key_compare my_compare; + random_level_generator_type my_rnd_generator; + node_ptr dummy_head; + + template + friend class concurrent_skip_list; + + std::atomic my_size; +}; // class concurrent_skip_list + +template +class concurrent_geometric_level_generator { +public: + static constexpr size_t max_level = MAX_LEVEL; + + concurrent_geometric_level_generator() : engines(time(NULL)) {} + + size_t operator()() { + return (distribution(engines.local()) % MAX_LEVEL) + 1; + } + +private: + tbb::enumerable_thread_specific engines; + std::geometric_distribution distribution; +}; + +} // namespace internal +} // namespace interface10 +} // namespace tbb + +#endif // __TBB_concurrent_skip_list_H diff --git a/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h b/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h index 2395aa08..3998c3fa 100644 --- a/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h +++ b/src/tbb/include/tbb/internal/_concurrent_unordered_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* Container implementations in this header are based on PPL implementations @@ -44,7 +40,17 @@ #include #endif +#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_COPY_DELETION_BROKEN + #define __TBB_UNORDERED_NODE_HANDLE_PRESENT 1 +#endif + +#include "_allocator_traits.h" #include "_tbb_hash_compare_impl.h" +#include "_template_helpers.h" + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT +#include "_node_handle_impl.h" +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT namespace tbb { namespace interface5 { @@ -124,6 +130,8 @@ class solist_iterator : public flist_iterator friend class split_ordered_list; template friend class solist_iterator; + template + friend class concurrent_unordered_base; template friend bool operator==( const solist_iterator &i, const solist_iterator &j ); template @@ -185,17 +193,20 @@ class split_ordered_list { public: typedef split_ordered_list self_type; - typedef typename Allocator::template rebind::other allocator_type; + + typedef typename tbb::internal::allocator_rebind::type allocator_type; + struct node; typedef node *nodeptr_t; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::value_type value_type; + typedef typename tbb::internal::allocator_traits::value_type value_type; + typedef typename tbb::internal::allocator_traits::size_type size_type; + typedef typename tbb::internal::allocator_traits::difference_type difference_type; + typedef typename tbb::internal::allocator_traits::pointer pointer; + typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; + // No support for reference/const_reference in allocator traits + typedef value_type& reference; + typedef const value_type& const_reference; typedef solist_iterator const_iterator; typedef solist_iterator iterator; @@ -220,6 +231,15 @@ class split_ordered_list return my_order_key; } + // get() and value() is a common interface for getting access to node`s element (required by node_handle) + value_type* storage() { + return reinterpret_cast(&my_element); + } + + value_type& value() { + return *storage(); + } + // Inserts the new element in the list in an atomic fashion nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) { @@ -564,27 +584,49 @@ class split_ordered_list } - // This erase function can handle both real and dummy nodes - void erase_node(raw_iterator previous, raw_const_iterator& where) - { + nodeptr_t erase_node_impl(raw_iterator previous, raw_const_iterator& where) { nodeptr_t pnode = (where++).get_node_ptr(); nodeptr_t prevnode = previous.get_node_ptr(); __TBB_ASSERT(prevnode->my_next == pnode, "Erase must take consecutive iterators"); prevnode->my_next = pnode->my_next; + return pnode; + } + // This erase function can handle both real and dummy nodes + void erase_node(raw_iterator previous, raw_const_iterator& where, + /*allow_destroy*/tbb::internal::true_type) + { + nodeptr_t pnode = erase_node_impl(previous, where); destroy_node(pnode); } + void erase_node(raw_iterator previous, raw_const_iterator& where, + /*allow_destroy*/tbb::internal::false_type) + { + erase_node_impl(previous, where); + } + + void erase_node(raw_iterator previous, raw_const_iterator& where) { + erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); + } + // Erase the element (previous node needs to be passed because this is a forward only list) - iterator erase_node(raw_iterator previous, const_iterator where) + template + iterator erase_node(raw_iterator previous, const_iterator where, AllowDestroy) { raw_const_iterator it = where; - erase_node(previous, it); + erase_node(previous, it, AllowDestroy()); my_element_count--; return get_iterator(first_real_iterator(it)); } + iterator erase_node(raw_iterator previous, const_iterator& where) { + return erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); + } + + + // Move all elements from the passed in split-ordered list to this one void move_all(self_type& source) { @@ -639,7 +681,7 @@ class split_ordered_list #endif } - typename allocator_type::template rebind::other my_node_allocator; // allocator object for nodes + typename tbb::internal::allocator_rebind::type my_node_allocator; // allocator object for nodes size_type my_element_count; // Total item count, not counting dummy nodes nodeptr_t my_head; // pointer to head node }; @@ -661,12 +703,15 @@ class concurrent_unordered_base : public Traits typedef typename Traits::allocator_type allocator_type; typedef typename hash_compare::hasher hasher; typedef typename hash_compare::key_equal key_equal; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; + + typedef typename tbb::internal::allocator_traits::size_type size_type; + typedef typename tbb::internal::allocator_traits::difference_type difference_type; + typedef typename tbb::internal::allocator_traits::pointer pointer; + typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; + // No support for reference/const_reference in allocator + typedef typename allocator_type::value_type& reference; + typedef const typename allocator_type::value_type& const_reference; + typedef split_ordered_list solist_t; typedef typename solist_t::nodeptr_t nodeptr_t; // Iterators that walk the entire split-order list, including dummy nodes @@ -676,12 +721,19 @@ class concurrent_unordered_base : public Traits typedef typename solist_t::const_iterator const_iterator; typedef iterator local_iterator; typedef const_iterator const_local_iterator; +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + typedef typename Traits::node_type node_type; +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT using Traits::my_hash_compare; using Traits::get_key; using Traits::allow_multimapping; static const size_type initial_bucket_number = 8; // Initial number of buckets + private: + template + friend class concurrent_unordered_base; + typedef std::pair pairii_t; typedef std::pair paircc_t; @@ -727,8 +779,10 @@ class concurrent_unordered_base : public Traits #if __TBB_CPP11_RVALUE_REF_PRESENT concurrent_unordered_base(concurrent_unordered_base&& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) + : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()), + my_maximum_bucket_size(float(initial_bucket_load)) { + my_number_of_buckets = initial_bucket_number; internal_init(); swap(right); } @@ -740,6 +794,8 @@ class concurrent_unordered_base : public Traits internal_init(); if (a == right.get_allocator()){ + my_number_of_buckets = initial_bucket_number; + my_maximum_bucket_size = float(initial_bucket_load); this->swap(right); }else{ my_maximum_bucket_size = right.my_maximum_bucket_size; @@ -820,6 +876,43 @@ class concurrent_unordered_base : public Traits internal_clear(); } +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + template + void internal_merge(SourceType& source) { + typedef typename SourceType::iterator source_iterator; + __TBB_STATIC_ASSERT((tbb::internal::is_same_type::value), + "Incompatible containers cannot be merged"); + + for(source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || find(get_key(*where)) == end()) { + std::pair extract_result = source.internal_extract(where); + + // If the insertion fails, it returns ownership of the node to extract_result.first + // extract_result.first remains valid node handle + if (!insert(std::move(extract_result.first)).second) { + raw_iterator next = extract_result.second; + raw_iterator current = next++; + + __TBB_ASSERT(extract_result.first.my_node->get_order_key() >= current.get_node_ptr()->get_order_key(), + "Wrong nodes order in source container"); + __TBB_ASSERT(next==source.my_solist.raw_end() || + extract_result.first.my_node->get_order_key() <= next.get_node_ptr()->get_order_key(), + "Wrong nodes order in source container"); + + size_t new_count = 0;// To use try_insert() + bool insert_result = + source.my_solist.try_insert(current, next, extract_result.first.my_node, &new_count).second; + __TBB_ASSERT_EX(insert_result, "Return to source must be successful. " + "Changing source container while merging is unsafe."); + } + extract_result.first.deactivate(); + } + } + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + public: allocator_type get_allocator() const { return my_solist.get_allocator(); @@ -956,7 +1049,8 @@ class concurrent_unordered_base : public Traits // Modifiers std::pair insert(const value_type& value) { - return internal_insert(value); + return internal_insert(value); } iterator insert(const_iterator, const value_type& value) { @@ -966,23 +1060,43 @@ class concurrent_unordered_base : public Traits #if __TBB_CPP11_RVALUE_REF_PRESENT std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); + return internal_insert(std::move(value)); } iterator insert(const_iterator, value_type&& value) { // Ignore hint return insert(std::move(value)).first; } +#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + std::pair insert(node_type&& nh) { + if (!nh.empty()) { + nodeptr_t handled_node = nh.my_node; + std::pair insert_result = + internal_insert + (handled_node->my_element, handled_node); + if (insert_result.second) + nh.deactivate(); + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert(const_iterator, node_type&& nh) { + return insert(std::move(nh)).first; + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT template std::pair emplace(Args&&... args) { nodeptr_t pnode = my_solist.create_node_v(tbb::internal::forward(args)...); - const sokey_t hashed_element_key = (sokey_t) my_hash_compare(get_key(pnode->my_element)); - const sokey_t order_key = split_order_key_regular(hashed_element_key); - pnode->init(order_key); - return internal_insert(pnode->my_element, pnode); + return internal_insert(pnode->my_element, pnode); } template @@ -990,9 +1104,8 @@ class concurrent_unordered_base : public Traits // Ignore hint return emplace(tbb::internal::forward(args)...).first; } +#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif // __TBB_CPP11_RVALUE_REF_PRESENT template void insert(Iterator first, Iterator last) { @@ -1024,9 +1137,21 @@ class concurrent_unordered_base : public Traits return item_count; } +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + node_type unsafe_extract(const_iterator where) { + return internal_extract(where).first; + } + + node_type unsafe_extract(const key_type& key) { + pairii_t where = equal_range(key); + if (where.first == end()) return node_type(); // element was not found + return internal_extract(where.first).first; + } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + void swap(concurrent_unordered_base& right) { if (this != &right) { - std::swap(my_hash_compare, right.my_hash_compare); // TODO: check what ADL meant here + std::swap(my_hash_compare, right.my_hash_compare); my_solist.swap(right.my_solist); internal_swap_buckets(right); std::swap(my_number_of_buckets, right.my_number_of_buckets); @@ -1258,7 +1383,7 @@ class concurrent_unordered_base : public Traits } // Insert an element in the hash given its value - template + template std::pair internal_insert(__TBB_FORWARDING_REF(ValueType) value, nodeptr_t pnode = NULL) { const key_type *pkey = &get_key(value); @@ -1283,6 +1408,11 @@ class concurrent_unordered_base : public Traits // If the value was moved, the known reference to key might be invalid pkey = &get_key(pnode->my_element); } + else + { + // Set new order_key to node + pnode->init(order_key); + } // Try to insert 'pnode' between 'previous' and 'where' std::pair result = my_solist.try_insert(previous, where, pnode, &new_count); @@ -1307,7 +1437,7 @@ class concurrent_unordered_base : public Traits else if (!allow_multimapping && solist_t::get_order_key(where) == order_key && !my_hash_compare(get_key(*where), *pkey)) // TODO: fix negation { // Element already in the list, return it - if (pnode) + if (pnode && AllowDestroy::value) my_solist.destroy_node(pnode); return std::pair(my_solist.get_iterator(where), false); } @@ -1353,14 +1483,33 @@ class concurrent_unordered_base : public Traits __TBB_ASSERT(previous != last, "Invalid head node"); // First node is a dummy node - for (raw_iterator where = previous; ; previous = where) { + for (raw_iterator where = previous; where != last; previous = where) { ++where; - if (where == last) - return end(); - else if (my_solist.get_iterator(where) == it) + if (my_solist.get_iterator(where) == it) return my_solist.erase_node(previous, it); } + return end(); + } + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT + std::pair internal_extract(const_iterator it) { + sokey_t hash_key = sokey_t(my_hash_compare(get_key(*it))); + raw_iterator previous = prepare_bucket(hash_key); + raw_iterator last = my_solist.raw_end(); + __TBB_ASSERT(previous != last, "Invalid head node"); + + for(raw_iterator where = previous; where != last; previous = where) { + ++where; + if (my_solist.get_iterator(where) == it) { + const_iterator result = it; + my_solist.erase_node(previous, it, /*allow_destroy*/tbb::internal::false_type()); + return std::pair( node_type(result.get_node_ptr()), + previous); + } + } + return std::pair(node_type(), end()); } +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT // Return the [begin, end) pair of iterators with the same key values. // This operation makes sense only if mapping is many-to-one. @@ -1468,7 +1617,7 @@ class concurrent_unordered_base : public Traits if (my_buckets[segment] == NULL) { size_type sz = segment_size(segment); raw_iterator * new_segment = my_allocator.allocate(sz); - std::memset((void*) new_segment, 0, sz*sizeof(raw_iterator)); + std::memset(static_cast(new_segment), 0, sz*sizeof(raw_iterator)); if (my_buckets[segment].compare_and_swap( new_segment, NULL) != NULL) my_allocator.deallocate(new_segment, sz); @@ -1503,7 +1652,7 @@ class concurrent_unordered_base : public Traits // Shared variables atomic my_number_of_buckets; // Current table size solist_t my_solist; // List where all the elements are kept - typename allocator_type::template rebind::other my_allocator; // Allocator object for segments + typename tbb::internal::allocator_rebind::type my_allocator; // Allocator object for segments float my_maximum_bucket_size; // Maximum size of the bucket atomic my_buckets[pointers_per_table]; // The segment table }; diff --git a/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h b/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h index 794960db..ffb63ce4 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_async_msg_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_async_msg_impl_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_body_impl.h b/src/tbb/include/tbb/internal/_flow_graph_body_impl.h index 54f37411..57264fd3 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_body_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_body_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_body_impl_H @@ -33,11 +29,46 @@ typedef tbb::internal::uint64_t tag_value; using tbb::internal::strip; +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + +template struct Policy {}; + +template struct has_policy; + +template +struct has_policy : + tbb::internal::bool_constant::value || + has_policy::value> {}; + +template +struct has_policy : + tbb::internal::bool_constant::value> {}; + +template +struct has_policy > : has_policy {}; + +#else + +template struct Policy {}; + +template +struct has_policy : tbb::internal::bool_constant::value> {}; + +template +struct has_policy > : has_policy {}; + +template +struct has_policy > : + tbb::internal::bool_constant::value || has_policy::value> {}; + +#endif + namespace graph_policy_namespace { struct rejecting { }; struct reserving { }; struct queueing { }; + struct lightweight { }; // K == type of field used for key-matching. Each tag-matching port will be provided // functor that, given an object accepted by the port, will return the @@ -52,6 +83,10 @@ namespace graph_policy_namespace { // old tag_matching join's new specifier typedef key_matching tag_matching; + // Aliases for Policy combinations + typedef interface10::internal::Policy queueing_lightweight; + typedef interface10::internal::Policy rejecting_lightweight; + } // namespace graph_policy_namespace // -------------- function_body containers ---------------------- @@ -232,13 +267,20 @@ class type_to_key_function_body_leaf : public type_to_key_funct //! A task that calls a node's forward_task function template< typename NodeType > -class forward_task_bypass : public task { +class forward_task_bypass : public graph_task { NodeType &my_node; public: - forward_task_bypass( NodeType &n ) : my_node(n) {} + forward_task_bypass( NodeType &n +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_node(n) {} task *execute() __TBB_override { task * new_task = my_node.forward_task(); @@ -250,14 +292,21 @@ class forward_task_bypass : public task { //! A task that calls a node's apply_body_bypass function, passing in an input of type Input // return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL template< typename NodeType, typename Input > -class apply_body_task_bypass : public task { +class apply_body_task_bypass : public graph_task { NodeType &my_node; Input my_input; public: - apply_body_task_bypass( NodeType &n, const Input &i ) : my_node(n), my_input(i) {} + apply_body_task_bypass( NodeType &n, const Input &i +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_node(n), my_input(i) {} task *execute() __TBB_override { task * next_task = my_node.apply_body_bypass( my_input ); @@ -268,7 +317,7 @@ class apply_body_task_bypass : public task { //! A task that calls a node's apply_body_bypass function with no input template< typename NodeType > -class source_task_bypass : public task { +class source_task_bypass : public graph_task { NodeType &my_node; @@ -291,13 +340,85 @@ struct empty_body { Output operator()( const Input & ) const { return Output(); } }; +template +class decrementer; + +template +class decrementer::value, void>::type + > : public receiver, tbb::internal::no_copy { + T* my_node; +protected: + + task* try_put_task( const DecrementType& value ) __TBB_override { + task* result = my_node->decrement_counter( value ); + if( !result ) + result = SUCCESSFULLY_ENQUEUED; + return result; + } + + graph& graph_reference() __TBB_override { + return my_node->my_graph; + } + + template friend class tbb::flow::interface11::limiter_node; + void reset_receiver( reset_flags f ) __TBB_override { +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + if (f & rf_clear_edges) + my_built_predecessors.clear(); +#else + tbb::internal::suppress_unused_warning( f ); +#endif + } + +public: + // Since decrementer does not make use of possibly unconstructed owner inside its + // constructor, my_node can be directly initialized with 'this' pointer passed from the + // owner, hence making method 'set_owner' needless. + decrementer() : my_node(NULL) {} + void set_owner( T *node ) { my_node = node; } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + spin_mutex my_mutex; + //! The predecessor type for this node + typedef typename receiver::predecessor_type predecessor_type; + + typedef internal::edge_container built_predecessors_type; + typedef typename built_predecessors_type::edge_list_type predecessor_list_type; + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + + void internal_add_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.add_edge( s ); + } + + void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.delete_edge(s); + } + + void copy_predecessors( predecessor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.copy_edges(v); + } + + size_t predecessor_count() __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + return my_built_predecessors.edge_count(); + } +protected: + built_predecessors_type my_built_predecessors; +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ +}; + template -class decrementer : public continue_receiver, tbb::internal::no_copy { +class decrementer : public continue_receiver, tbb::internal::no_copy { T *my_node; task *execute() __TBB_override { - return my_node->decrement_counter(); + return my_node->decrement_counter( 1 ); } protected: @@ -310,7 +431,15 @@ class decrementer : public continue_receiver, tbb::internal::no_copy { typedef continue_msg input_type; typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) : continue_receiver( number_of_predecessors ) { } + decrementer( int number_of_predecessors = 0 ) + : continue_receiver( + __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, tbb::flow::internal::no_priority) + ) + // Since decrementer does not make use of possibly unconstructed owner inside its + // constructor, my_node can be directly initialized with 'this' pointer passed from the + // owner, hence making method 'set_owner' needless. + , my_node(NULL) + {} void set_owner( T *node ) { my_node = node; } }; diff --git a/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h b/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h index 7fac84e8..f3081b6f 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_cache_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_cache_impl_H @@ -57,12 +53,12 @@ class node_cache { void clear() { while( !my_q.empty()) (void)my_q.pop(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_predecessors.clear(); #endif } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef edge_container built_predecessors_type; built_predecessors_type &built_predecessors() { return my_built_predecessors; } @@ -86,14 +82,14 @@ class node_cache { typename mutex_type::scoped_lock lock(my_mutex); return (size_t)(my_built_predecessors.edge_count()); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: typedef M mutex_type; mutex_type my_mutex; std::queue< T * > my_q; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type my_built_predecessors; #endif @@ -189,7 +185,7 @@ class predecessor_cache : public node_cache< sender, M > { protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION using node_cache< predecessor_type, M >::my_built_predecessors; #endif successor_type *my_owner; @@ -290,7 +286,7 @@ class successor_cache : tbb::internal::no_copy { typedef sender owner_type; #endif // __TBB_PREVIEW_ASYNC_MSG typedef std::list< pointer_type > successors_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_successors; #endif successors_type my_successors; @@ -298,7 +294,7 @@ class successor_cache : tbb::internal::no_copy { owner_type *my_owner; public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename edge_container::edge_list_type successor_list_type; edge_container &built_successors() { return my_built_successors; } @@ -323,7 +319,7 @@ class successor_cache : tbb::internal::no_copy { return my_built_successors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ successor_cache( ) : my_owner(NULL) {} @@ -354,7 +350,7 @@ class successor_cache : tbb::internal::no_copy { void clear() { my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_successors.clear(); #endif } @@ -381,7 +377,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { #endif // __TBB_PREVIEW_ASYNC_MSG typedef std::list< pointer_type > successors_type; successors_type my_successors; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_successors; typedef edge_container::edge_list_type successor_list_type; #endif @@ -390,7 +386,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container &built_successors() { return my_built_successors; } @@ -414,7 +410,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { return my_built_successors.edge_count(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ successor_cache( ) : my_owner(NULL) {} @@ -452,7 +448,7 @@ class successor_cache< continue_msg > : tbb::internal::no_copy { void clear() { my_successors.clear(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION my_built_successors.clear(); #endif } @@ -508,6 +504,40 @@ class broadcast_cache : public successor_cache { return last_task; } + // call try_put_task and return list of received tasks +#if __TBB_PREVIEW_ASYNC_MSG + template + bool gather_successful_try_puts( const X &t, task_list &tasks ) { +#else + bool gather_successful_try_puts( const T &t, task_list &tasks ) { +#endif // __TBB_PREVIEW_ASYNC_MSG + bool upgraded = true; + bool is_at_least_one_put_successful = false; + typename mutex_type::scoped_lock l(this->my_mutex, upgraded); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + task * new_task = (*i)->try_put_task(t); + if(new_task) { + ++i; + if(new_task != SUCCESSFULLY_ENQUEUED) { + tasks.push_back(*new_task); + } + is_at_least_one_put_successful = true; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + if (!upgraded) { + l.upgrade_to_writer(); + upgraded = true; + } + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return is_at_least_one_put_successful; + } }; //! A cache of successors that are put in a round-robin fashion diff --git a/src/tbb/include/tbb/internal/_flow_graph_impl.h b/src/tbb/include/tbb/internal/_flow_graph_impl.h index ffdd57be..a4b96d10 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_impl_H @@ -26,6 +22,10 @@ #include "../task_arena.h" #include "../flow_graph_abstractions.h" +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#include "../concurrent_priority_queue.h" +#endif + #include #if TBB_DEPRECATED_FLOW_ENQUEUE @@ -34,17 +34,54 @@ #define FLOW_SPAWN(a) tbb::task::spawn((a)) #endif +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) expr +#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) , priority +#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1, priority +#else +#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) +#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) +#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1 +#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) expr +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1, arg2 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg3, arg4 +#else +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg2 +#endif // TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + namespace tbb { namespace flow { namespace internal { static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +typedef unsigned int node_priority_t; +static const node_priority_t no_priority = node_priority_t(0); +#endif } namespace interface10 { using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +using tbb::flow::internal::node_priority_t; +using tbb::flow::internal::no_priority; +//! Base class for tasks generated by graph nodes. +struct graph_task : public task { + graph_task( node_priority_t node_priority = no_priority ) : priority( node_priority ) {} + node_priority_t priority; +}; +#else +typedef task graph_task; +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ + + class graph; class graph_node; @@ -127,9 +164,41 @@ namespace internal { void activate_graph(graph& g); void deactivate_graph(graph& g); bool is_graph_active(graph& g); +tbb::task& prioritize_task(graph& g, tbb::task& arena_task); void spawn_in_graph_arena(graph& g, tbb::task& arena_task); +void enqueue_in_graph_arena(graph &g, tbb::task& arena_task); void add_task_to_graph_reset_list(graph& g, tbb::task *tp); -template void execute_in_graph_arena(graph& g, F& f); + +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +struct graph_task_comparator { + bool operator()(const graph_task* left, const graph_task* right) { + return left->priority < right->priority; + } +}; + +typedef tbb::concurrent_priority_queue graph_task_priority_queue_t; + +class priority_task_selector : public task { +public: + priority_task_selector(graph_task_priority_queue_t& priority_queue) + : my_priority_queue(priority_queue) {} + task* execute() __TBB_override { + graph_task* t = NULL; + bool result = my_priority_queue.try_pop(t); + __TBB_ASSERT_EX( result, "Number of critical tasks for scheduler and tasks" + " in graph's priority queue mismatched" ); + __TBB_ASSERT( t && t != SUCCESSFULLY_ENQUEUED, + "Incorrect task submitted to graph priority queue" ); + __TBB_ASSERT( t->priority != tbb::flow::internal::no_priority, + "Tasks from graph's priority queue must have priority" ); + task* t_next = t->execute(); + task::destroy(*t); + return t_next; + } +private: + graph_task_priority_queue_t& my_priority_queue; +}; +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ } @@ -139,9 +208,16 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { friend class graph_node; template< typename Body > - class run_task : public task { + class run_task : public graph_task { public: - run_task(Body& body) : my_body(body) {} + run_task(Body& body +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , node_priority_t node_priority = no_priority + ) : graph_task(node_priority), +#else + ) : +#endif + my_body(body) { } tbb::task *execute() __TBB_override { my_body(); return NULL; @@ -151,7 +227,7 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { }; template< typename Receiver, typename Body > - class run_and_put_task : public task { + class run_and_put_task : public graph_task { public: run_and_put_task(Receiver &r, Body& body) : my_receiver(r), my_body(body) {} tbb::task *execute() __TBB_override { @@ -206,13 +282,7 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { //! Destroys the graph. /** Calls wait_for_all, then destroys the root task and context. */ - ~graph() { - wait_for_all(); - my_root_task->set_ref_count(0); - tbb::task::destroy(*my_root_task); - if (own_context) delete my_context; - delete my_task_arena; - } + ~graph(); #if TBB_PREVIEW_FLOW_GRAPH_TRACE void set_name(const char *name); @@ -305,17 +375,17 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { // Graph iterator constructors //! start iterator - iterator begin() { return iterator(this, true); } + iterator begin(); //! end iterator - iterator end() { return iterator(this, false); } + iterator end(); //! start const iterator - const_iterator begin() const { return const_iterator(this, true); } + const_iterator begin() const; //! end const iterator - const_iterator end() const { return const_iterator(this, false); } + const_iterator end() const; //! start const iterator - const_iterator cbegin() const { return const_iterator(this, true); } + const_iterator cbegin() const; //! end const iterator - const_iterator cend() const { return const_iterator(this, false); } + const_iterator cend() const; //! return status of graph execution bool is_cancelled() { return cancelled; } @@ -341,12 +411,17 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { tbb::task_arena* my_task_arena; +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + internal::graph_task_priority_queue_t my_priority_queue; +#endif + friend void internal::activate_graph(graph& g); friend void internal::deactivate_graph(graph& g); friend bool internal::is_graph_active(graph& g); + friend tbb::task& internal::prioritize_task(graph& g, tbb::task& arena_task); friend void internal::spawn_in_graph_arena(graph& g, tbb::task& arena_task); + friend void internal::enqueue_in_graph_arena(graph &g, tbb::task& arena_task); friend void internal::add_task_to_graph_reset_list(graph& g, tbb::task *tp); - template friend void internal::execute_in_graph_arena(graph& g, F& f); friend class tbb::interface7::internal::task_arena_base; @@ -361,18 +436,15 @@ class graph_node : tbb::internal::no_copy { graph& my_graph; graph_node *next, *prev; public: - explicit graph_node(graph& g) : my_graph(g) { - my_graph.register_node(this); - } - virtual ~graph_node() { - my_graph.remove_node(this); - } + explicit graph_node(graph& g); + + virtual ~graph_node(); #if TBB_PREVIEW_FLOW_GRAPH_TRACE virtual void set_name(const char *name) = 0; #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION virtual void extract() = 0; #endif @@ -395,19 +467,43 @@ inline bool is_graph_active(graph& g) { return g.my_is_active; } -//! Executes custom functor inside graph arena -template -inline void execute_in_graph_arena(graph& g, F& f) { - if (is_graph_active(g)) { - __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); - g.my_task_arena->execute(f); +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +inline tbb::task& prioritize_task(graph& g, tbb::task& t) { + task* critical_task = &t; + // TODO: change flow graph's interfaces to work with graph_task type instead of tbb::task. + graph_task* gt = static_cast(&t); + if( gt->priority != no_priority ) { + //! Non-preemptive priority pattern. The original task is submitted as a work item to the + //! priority queue, and a new critical task is created to take and execute a work item with + //! the highest known priority. The reference counting responsibility is transferred (via + //! allocate_continuation) to the new task. + critical_task = new( gt->allocate_continuation() ) priority_task_selector(g.my_priority_queue); + tbb::internal::make_critical( *critical_task ); + g.my_priority_queue.push(gt); } + return *critical_task; +} +#else +inline tbb::task& prioritize_task(graph&, tbb::task& t) { + return t; } +#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ //! Spawns a task inside graph arena inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { - graph::spawn_functor s_fn(arena_task); - execute_in_graph_arena(g, s_fn); + if (is_graph_active(g)) { + graph::spawn_functor s_fn(prioritize_task(g, arena_task)); + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); + g.my_task_arena->execute(s_fn); + } +} + +//! Enqueues a task inside graph arena +inline void enqueue_in_graph_arena(graph &g, tbb::task& arena_task) { + if (is_graph_active(g)) { + __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); + task::enqueue(prioritize_task(g, arena_task), *g.my_task_arena); + } } inline void add_task_to_graph_reset_list(graph& g, tbb::task *tp) { diff --git a/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h index c8d4fd4e..332467dd 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_indexer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_indexer_impl_H @@ -54,7 +50,7 @@ namespace internal { indexer_helper::reset_inputs(my_input, f); tbb::flow::get(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract(InputTuple &my_input) { indexer_helper::extract(my_input); @@ -75,7 +71,7 @@ namespace internal { static inline void reset_inputs(InputTuple &my_input, reset_flags f) { tbb::flow::get<0>(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract(InputTuple &my_input) { tbb::flow::get<0>(my_input).extract_receiver(); @@ -89,24 +85,24 @@ namespace internal { void* my_indexer_ptr; typedef task* (* forward_function_ptr)(T const &, void* ); forward_function_ptr my_try_put_task; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION spin_mutex my_pred_mutex; typedef typename receiver::built_predecessors_type built_predecessors_type; built_predecessors_type my_built_predecessors; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ graph* my_graph; public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION indexer_input_port() : my_pred_mutex(), my_graph(NULL) {} indexer_input_port( const indexer_input_port & other) : receiver(), my_pred_mutex(), my_graph(other.my_graph) { } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ void set_up(void* p, forward_function_ptr f, graph& g) { my_indexer_ptr = p; my_try_put_task = f; my_graph = &g; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename receiver::predecessor_type predecessor_type; @@ -128,7 +124,7 @@ namespace internal { spin_mutex::scoped_lock l(my_pred_mutex); my_built_predecessors.copy_edges(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: template< typename R, typename B > friend class run_and_put_task; template friend class internal::broadcast_cache; @@ -142,13 +138,13 @@ namespace internal { } public: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void reset_receiver(reset_flags f) __TBB_override { if(f&rf_clear_edges) my_built_predecessors.clear(); } #else void reset_receiver(reset_flags /*f*/) __TBB_override { } #endif -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract_receiver() { my_built_predecessors.receiver_extract(*this); } #endif }; @@ -180,7 +176,7 @@ namespace internal { typedef StructTypes tuple_types; typedef typename sender::successor_type successor_type; typedef indexer_node_FE input_ports_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -188,7 +184,7 @@ namespace internal { private: // ----------- Aggregator ------------ enum op_type { reg_succ, rem_succ, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy #endif @@ -202,7 +198,7 @@ namespace internal { output_type const *my_arg; successor_type *my_succ; task *bypass_t; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; successor_list_type *succv; #endif @@ -239,7 +235,7 @@ namespace internal { __TBB_store_with_release(current->status, SUCCEEDED); // return of try_put_task actual return value } break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_succ: my_successors.internal_add_built_successor(*(current->my_succ)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -256,7 +252,7 @@ namespace internal { my_successors.copy_successors(*(current->succv)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -292,7 +288,7 @@ namespace internal { return op_data.bypass_t; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } @@ -321,7 +317,7 @@ namespace internal { my_successors.built_successors().sender_extract(*this); indexer_helper::extract(this->my_inputs); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: void reset_node(reset_flags f) __TBB_override { if(f & rf_clear_edges) { diff --git a/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h index 85d2686d..9ac4dbbb 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_item_buffer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_item_buffer_impl_H @@ -144,7 +140,7 @@ namespace internal { return get_my_item(my_tail - 1); } - // following methods are for reservation of the front of a bufffer. + // following methods are for reservation of the front of a buffer. void reserve_item(size_type i) { __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); item(i).second = reserved_item; } void release_item(size_type i) { __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); item(i).second = has_item; } diff --git a/src/tbb/include/tbb/internal/_flow_graph_join_impl.h b/src/tbb/include/tbb/internal/_flow_graph_join_impl.h index 8fc32a9a..4ccaef9f 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_join_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_join_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_join_impl_H @@ -126,7 +122,7 @@ namespace internal { tbb::flow::get(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract_inputs(InputTuple &my_input) { join_helper::extract_inputs(my_input); @@ -200,7 +196,7 @@ namespace internal { tbb::flow::get<0>(my_input).reset_receiver(f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION template static inline void extract_inputs(InputTuple &my_input) { tbb::flow::get<0>(my_input).extract_receiver(); @@ -214,18 +210,17 @@ namespace internal { public: typedef T input_type; typedef typename receiver::predecessor_type predecessor_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::predecessor_list_type predecessor_list_type; typedef typename receiver::built_predecessors_type built_predecessors_type; #endif private: // ----------- Aggregator ------------ enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef reserving_port class_type; class reserving_port_operation : public aggregated_operation { @@ -234,7 +229,7 @@ namespace internal { union { T *my_arg; predecessor_type *my_pred; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; predecessor_list_type *plist; #endif @@ -294,7 +289,7 @@ namespace internal { my_predecessors.try_consume( ); __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_predecessors.internal_add_built_predecessor(*(current->my_pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -311,7 +306,7 @@ namespace internal { my_predecessors.copy_predecessors(*(current->plist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -382,7 +377,7 @@ namespace internal { my_aggregator.execute(&op_data); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } void internal_add_built_predecessor(predecessor_type &src) __TBB_override { reserving_port_operation op_data(src, add_blt_pred); @@ -410,7 +405,7 @@ namespace internal { my_predecessors.built_predecessors().receiver_extract(*this); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ void reset_receiver( reset_flags f) __TBB_override { if(f & rf_clear_edges) my_predecessors.clear(); @@ -433,7 +428,7 @@ namespace internal { typedef T input_type; typedef typename receiver::predecessor_type predecessor_type; typedef queueing_port class_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif @@ -441,18 +436,17 @@ namespace internal { // ----------- Aggregator ------------ private: enum op_type { get__item, res_port, try__put_task -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class queueing_port_operation : public aggregated_operation { public: char type; T my_val; T *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION predecessor_type *pred; size_t cnt_val; predecessor_list_type *plist; @@ -513,7 +507,7 @@ namespace internal { } __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_built_predecessors.add_edge(*(current->pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -530,7 +524,7 @@ namespace internal { my_built_predecessors.copy_edges(*(current->plist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -585,7 +579,7 @@ namespace internal { return; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } void internal_add_built_predecessor(predecessor_type &p) __TBB_override { @@ -616,12 +610,12 @@ namespace internal { item_buffer::reset(); my_built_predecessors.receiver_extract(*this); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ void reset_receiver(reset_flags f) __TBB_override { tbb::internal::suppress_unused_warning(f); item_buffer::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION if (f & rf_clear_edges) my_built_predecessors.clear(); #endif @@ -629,7 +623,7 @@ namespace internal { private: forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_predecessors; #endif }; // queueing_port @@ -667,7 +661,7 @@ namespace internal { typedef typename TraitsType::TtoK type_to_key_func_type; typedef typename TraitsType::KHash hash_compare_type; typedef hash_buffer< key_type, input_type, type_to_key_func_type, hash_compare_type > buffer_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename receiver::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif @@ -675,18 +669,17 @@ namespace internal { // ----------- Aggregator ------------ private: enum op_type { try__put, get__item, res_port -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class key_matching_port_operation : public aggregated_operation { public: char type; input_type my_val; input_type *my_arg; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION predecessor_type *pred; size_t cnt_val; predecessor_list_type *plist; @@ -729,7 +722,7 @@ namespace internal { this->delete_with_key(my_join->current_key); __TBB_store_with_release(current->status, SUCCEEDED); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: my_built_predecessors.add_edge(*(current->pred)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -801,7 +794,7 @@ namespace internal { return op_data.status == SUCCEEDED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } void internal_add_built_predecessor(predecessor_type &p) __TBB_override { @@ -837,7 +830,7 @@ namespace internal { return; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract_receiver() { buffer_type::reset(); my_built_predecessors.receiver_extract(*this); @@ -846,7 +839,7 @@ namespace internal { void reset_receiver(reset_flags f ) __TBB_override { tbb::internal::suppress_unused_warning(f); buffer_type::reset(); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION if (f & rf_clear_edges) my_built_predecessors.clear(); #endif @@ -856,7 +849,7 @@ namespace internal { // my_join forwarding base used to count number of inputs that // received key. matching_forwarding_base *my_join; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION edge_container my_built_predecessors; #endif }; // key_matching_port @@ -917,7 +910,7 @@ namespace internal { join_helper::reset_inputs(my_inputs, f); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract( ) { // called outside of parallel contexts ports_with_no_inputs = N; @@ -998,7 +991,7 @@ namespace internal { join_helper::reset_inputs(my_inputs, f ); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { reset_port_count(); join_helper::extract_inputs(my_inputs); @@ -1067,7 +1060,6 @@ namespace internal { // and the output_buffer_type base class private: enum op_type { res_count, inc_count, may_succeed, try_make }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_FE, InputTuple, OutputTuple> class_type; class key_matching_FE_operation : public aggregated_operation { @@ -1222,7 +1214,7 @@ namespace internal { output_buffer_type::reset(); } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { // called outside of parallel contexts join_helper::extract_inputs(my_inputs); @@ -1274,7 +1266,7 @@ namespace internal { using input_ports_type::try_to_make_tuple; using input_ports_type::tuple_accepted; using input_ports_type::tuple_rejected; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -1282,11 +1274,10 @@ namespace internal { private: // ----------- Aggregator ------------ enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_base class_type; class join_node_base_operation : public aggregated_operation { @@ -1295,7 +1286,7 @@ namespace internal { union { output_type *my_arg; successor_type *my_succ; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; successor_list_type *slist; #endif @@ -1370,7 +1361,7 @@ namespace internal { forwarder_busy = false; } break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_succ: my_successors.internal_add_built_successor(*(current->my_succ)); __TBB_store_with_release(current->status, SUCCEEDED); @@ -1387,7 +1378,7 @@ namespace internal { my_successors.copy_successors(*(current->slist)); __TBB_store_with_release(current->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -1432,7 +1423,7 @@ namespace internal { return op_data.status == SUCCEEDED; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } void internal_add_built_successor( successor_type &r) __TBB_override { @@ -1456,9 +1447,9 @@ namespace internal { op_data.slist = &l; my_aggregator.execute(&op_data); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() __TBB_override { input_ports_type::extract(); my_successors.built_successors().sender_extract(*this); diff --git a/src/tbb/include/tbb/internal/_flow_graph_node_impl.h b/src/tbb/include/tbb/internal/_flow_graph_node_impl.h index 65a55b2d..215777f3 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_node_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_node_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_node_impl_H @@ -61,15 +57,15 @@ namespace internal { //! Input and scheduling for a function node that takes a type Input as input // The only up-ref is apply_body_impl, which should implement the function // call and any handling of the result. - template< typename Input, typename A, typename ImplType > + template< typename Input, typename Policy, typename A, typename ImplType > class function_input_base : public receiver, tbb::internal::no_assign { - enum op_type {reg_pred, rem_pred, app_body, try_fwd, tryput_bypass, app_body_bypass -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy // create vector copies of preds and succs #endif }; - typedef function_input_base class_type; + typedef function_input_base class_type; public: @@ -79,25 +75,32 @@ namespace internal { typedef predecessor_cache predecessor_cache_type; typedef function_input_queue input_queue_type; typedef typename A::template rebind< input_queue_type >::other queue_allocator_type; + __TBB_STATIC_ASSERT(!((internal::has_policy::value) && (internal::has_policy::value)), + "queueing and rejecting policies can't be specified simultaneously"); -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; typedef typename receiver::predecessor_list_type predecessor_list_type; #endif //! Constructor for function_input_base - function_input_base( graph &g, size_t max_concurrency, input_queue_type *q = NULL) - : my_graph_ref(g), my_max_concurrency(max_concurrency), my_concurrency(0), - my_queue(q), forwarder_busy(false) { + function_input_base( + graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(size_t max_concurrency, node_priority_t priority) + ) : my_graph_ref(g), my_max_concurrency(max_concurrency) + , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(priority)) + , my_queue(!internal::has_policy::value ? new input_queue_type() : NULL) + , forwarder_busy(false) + { my_predecessors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); } //! Copy constructor - function_input_base( const function_input_base& src, input_queue_type *q = NULL) : - receiver(), tbb::internal::no_assign(), - my_graph_ref(src.my_graph_ref), my_max_concurrency(src.my_max_concurrency), - my_concurrency(0), my_queue(q), forwarder_busy(false) + function_input_base( const function_input_base& src) + : receiver(), tbb::internal::no_assign() + , my_graph_ref(src.my_graph_ref), my_max_concurrency(src.my_max_concurrency) + , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(src.my_priority)) + , my_queue(src.my_queue ? new input_queue_type() : NULL), forwarder_busy(false) { my_predecessors.set_owner(this); my_aggregator.initialize_handler(handler_type(this)); @@ -111,18 +114,8 @@ namespace internal { if ( my_queue ) delete my_queue; } - //! Put to the node, returning a task if available - task * try_put_task( const input_type &t ) __TBB_override { - if ( my_max_concurrency == 0 ) { - return create_body_task( t ); - } else { - operation_type op_data(t, tryput_bypass); - my_aggregator.execute(&op_data); - if(op_data.status == internal::SUCCEEDED) { - return op_data.bypass_t; - } - return NULL; - } + task* try_put_task( const input_type& t) __TBB_override { + return try_put_task_impl(t, internal::has_policy()); } //! Adds src to the list of cached predecessors. @@ -141,7 +134,7 @@ namespace internal { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION //! Adds to list of predecessors added by make_edge void internal_add_built_predecessor( predecessor_type &src) __TBB_override { operation_type op_data(add_blt_pred); @@ -171,7 +164,7 @@ namespace internal { built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ protected: @@ -187,6 +180,7 @@ namespace internal { graph& my_graph_ref; const size_t my_max_concurrency; size_t my_concurrency; + __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) input_queue_type *my_queue; predecessor_cache my_predecessors; @@ -201,6 +195,12 @@ namespace internal { return my_graph_ref; } + task* try_get_postponed_task(const input_type& i) { + operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + private: friend class apply_body_task_bypass< class_type, input_type >; @@ -212,10 +212,10 @@ namespace internal { union { input_type *elem; predecessor_type *r; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION size_t cnt_val; predecessor_list_type *predv; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ }; tbb::task *bypass_t; operation_type(const input_type& e, op_type t) : @@ -228,7 +228,7 @@ namespace internal { friend class internal::aggregating_functor; aggregator< handler_type, operation_type > my_aggregator; - task* create_and_spawn_task(bool spawn) { + task* perform_queued_requests() { task* new_task = NULL; if(my_queue) { if(!my_queue->empty()) { @@ -245,13 +245,6 @@ namespace internal { new_task = create_body_task(i); } } - //! Spawns a task that applies a body - // task == NULL => g.reset(), which shouldn't occur in concurrent context - if(spawn && new_task) { - internal::spawn_in_graph_arena(graph_reference(), *new_task); - new_task = SUCCESSFULLY_ENQUEUED; - } - return new_task; } void handle_operations(operation_type *op_list) { @@ -272,27 +265,27 @@ namespace internal { my_predecessors.remove(*(tmp->r)); __TBB_store_with_release(tmp->status, SUCCEEDED); break; - case app_body: - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (my_concurrencybypass_t = NULL; __TBB_ASSERT(my_max_concurrency != 0, NULL); --my_concurrency; if(my_concurrencybypass_t = create_and_spawn_task(/*spawn=*/false); + tmp->bypass_t = perform_queued_requests(); __TBB_store_with_release(tmp->status, SUCCEEDED); } break; case tryput_bypass: internal_try_put_task(tmp); break; case try_fwd: internal_forward(tmp); break; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES + case occupy_concurrency: + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + __TBB_store_with_release(tmp->status, SUCCEEDED); + } else { + __TBB_store_with_release(tmp->status, FAILED); + } + break; +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION case add_blt_pred: { my_predecessors.internal_add_built_predecessor(*(tmp->r)); __TBB_store_with_release(tmp->status, SUCCEEDED); @@ -310,7 +303,7 @@ namespace internal { my_predecessors.copy_predecessors( *(tmp->predv) ); __TBB_store_with_release(tmp->status, SUCCEEDED); break; -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ } } } @@ -332,11 +325,11 @@ namespace internal { } } - //! Tries to spawn bodies if available and if concurrency allows + //! Creates tasks for postponed messages if available and if concurrency allows void internal_forward(operation_type *op) { op->bypass_t = NULL; if (my_concurrency < my_max_concurrency || !my_max_concurrency) - op->bypass_t = create_and_spawn_task(/*spawn=*/false); + op->bypass_t = perform_queued_requests(); if(op->bypass_t) __TBB_store_with_release(op->status, SUCCEEDED); else { @@ -345,39 +338,61 @@ namespace internal { } } + task* internal_try_put_bypass( const input_type& t ) { + operation_type op_data(t, tryput_bypass); + my_aggregator.execute(&op_data); + if( op_data.status == internal::SUCCEEDED ) { + return op_data.bypass_t; + } + return NULL; + } + + task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::true_type ) { + if( my_max_concurrency == 0 ) { + return apply_body_bypass(t); + } else { + operation_type check_op(t, occupy_concurrency); + my_aggregator.execute(&check_op); + if( check_op.status == internal::SUCCEEDED ) { + return apply_body_bypass(t); + } + return internal_try_put_bypass(t); + } + } + + task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::false_type ) { + if( my_max_concurrency == 0 ) { + return create_body_task(t); + } else { + return internal_try_put_bypass(t); + } + } + //! Applies the body to the provided input // then decides if more work is available - task * apply_body_bypass( input_type &i ) { - task * new_task = static_cast(this)->apply_body_impl_bypass(i); - if ( my_max_concurrency != 0 ) { - operation_type op_data(app_body_bypass); // tries to pop an item or get_item, enqueues another apply_body - my_aggregator.execute(&op_data); - // workaround for icc bug - tbb::task *ttask = op_data.bypass_t; - new_task = combine_tasks(my_graph_ref, new_task, ttask); - } - return new_task; + task * apply_body_bypass( const input_type &i ) { + return static_cast(this)->apply_body_impl_bypass(i); } //! allocates a task to apply a body inline task * create_body_task( const input_type &input ) { - return (internal::is_graph_active(my_graph_ref)) ? - new(task::allocate_additional_child_of(*(my_graph_ref.root_task()))) - apply_body_task_bypass < class_type, input_type >(*this, input) : - NULL; + new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) + apply_body_task_bypass < class_type, input_type >( + *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(input, my_priority)) + : NULL; } //! This is executed by an enqueued task, the "forwarder" - task *forward_task() { + task* forward_task() { operation_type op_data(try_fwd); - task *rval = NULL; + task* rval = NULL; do { op_data.status = WAIT; my_aggregator.execute(&op_data); if(op_data.status == SUCCEEDED) { - // workaround for icc bug - tbb::task *ttask = op_data.bypass_t; + task* ttask = op_data.bypass_t; + __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, NULL ); rval = combine_tasks(my_graph_ref, rval, ttask); } } while (op_data.status == SUCCEEDED); @@ -386,8 +401,9 @@ namespace internal { inline task *create_forward_task() { return (internal::is_graph_active(my_graph_ref)) ? - new(task::allocate_additional_child_of(*(my_graph_ref.root_task()))) forward_task_bypass< class_type >(*this) : - NULL; + new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) + forward_task_bypass< class_type >( __TBB_FLOW_GRAPH_PRIORITY_ARG1(*this, my_priority) ) + : NULL; } //! Spawns a task that calls forward() @@ -401,27 +417,29 @@ namespace internal { //! Implements methods for a function node that takes a type Input as input and sends // a type Output to its successors. - template< typename Input, typename Output, typename A> - class function_input : public function_input_base > { + template< typename Input, typename Output, typename Policy, typename A> + class function_input : public function_input_base > { public: typedef Input input_type; typedef Output output_type; typedef function_body function_body_type; - typedef function_input my_class; - typedef function_input_base base_type; + typedef function_input my_class; + typedef function_input_base base_type; typedef function_input_queue input_queue_type; // constructor template - function_input( graph &g, size_t max_concurrency, Body& body, input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { + function_input( + graph &g, size_t max_concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } //! Copy constructor - function_input( const function_input& src, input_queue_type *q = NULL ) : - base_type(src, q), + function_input( const function_input& src ) : + base_type(src), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ) { } @@ -437,18 +455,52 @@ namespace internal { return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); } - task * apply_body_impl_bypass( const input_type &i) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE + output_type apply_body_impl( const input_type& i) { // There is an extra copied needed to capture the // body execution without the try_put tbb::internal::fgt_begin_body( my_body ); output_type v = (*my_body)(i); tbb::internal::fgt_end_body( my_body ); - task * new_task = successors().try_put_task( v ); + return v; + } + + //TODO: consider moving into the base class + task * apply_body_impl_bypass( const input_type &i) { + output_type v = apply_body_impl(i); +#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER + task* successor_task = successors().try_put_task(v); +#endif + task* postponed_task = NULL; + if( base_type::my_max_concurrency != 0 ) { + postponed_task = base_type::try_get_postponed_task(i); + __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, NULL ); + } +#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER + graph& g = base_type::my_graph_ref; + return combine_tasks(g, successor_task, postponed_task); #else - task * new_task = successors().try_put_task( (*my_body)(i) ); + if( postponed_task ) { + // make the task available for other workers since we do not know successors' + // execution policy + internal::spawn_in_graph_arena(base_type::graph_reference(), *postponed_task); + } + task* successor_task = successors().try_put_task(v); +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ #endif - return new_task; + if(internal::has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + if(!successor_task) { + // Return confirmative status since current + // node's body has been executed anyway + successor_task = SUCCESSFULLY_ENQUEUED; + } + } + return successor_task; +#endif /* TBB_DEPRECATED_MESSAGE_FLOW_ORDER */ } protected: @@ -491,7 +543,7 @@ namespace internal { } }; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION // helper templates to extract the output ports of an multifunction_node from graph template struct extract_element { template static void extract_this(P &p) { @@ -509,32 +561,29 @@ namespace internal { //! Implements methods for a function node that takes a type Input as input // and has a tuple of output ports specified. - template< typename Input, typename OutputPortSet, typename A> - class multifunction_input : public function_input_base > { + template< typename Input, typename OutputPortSet, typename Policy, typename A> + class multifunction_input : public function_input_base > { public: static const int N = tbb::flow::tuple_size::value; typedef Input input_type; typedef OutputPortSet output_ports_type; typedef multifunction_body multifunction_body_type; - typedef multifunction_input my_class; - typedef function_input_base base_type; + typedef multifunction_input my_class; + typedef function_input_base base_type; typedef function_input_queue input_queue_type; // constructor template - multifunction_input( - graph &g, - size_t max_concurrency, - Body& body, - input_queue_type *q = NULL ) : - base_type(g, max_concurrency, q), - my_body( new internal::multifunction_body_leaf(body) ), - my_init_body( new internal::multifunction_body_leaf(body) ) { + multifunction_input(graph &g, size_t max_concurrency, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) + , my_body( new internal::multifunction_body_leaf(body) ) + , my_init_body( new internal::multifunction_body_leaf(body) ) { } //! Copy constructor - multifunction_input( const multifunction_input& src, input_queue_type *q = NULL ) : - base_type(src, q), + multifunction_input( const multifunction_input& src ) : + base_type(src), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ) { } @@ -552,18 +601,22 @@ namespace internal { // for multifunction nodes we do not have a single successor as such. So we just tell // the task we were successful. + //TODO: consider moving common parts with implementation in function_input into separate function task * apply_body_impl_bypass( const input_type &i) { tbb::internal::fgt_begin_body( my_body ); (*my_body)(i, my_output_ports); tbb::internal::fgt_end_body( my_body ); - task * new_task = SUCCESSFULLY_ENQUEUED; - return new_task; + task* ttask = NULL; + if(base_type::my_max_concurrency != 0) { + ttask = base_type::try_get_postponed_task(i); + } + return ttask ? ttask : SUCCESSFULLY_ENQUEUED; } output_ports_type &output_ports(){ return my_output_ports; } protected: -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION void extract() { extract_element::extract_this(my_output_ports); } @@ -621,7 +674,7 @@ namespace internal { }; //! Implements methods for an executable node that takes continue_msg as input - template< typename Output > + template< typename Output, typename Policy> class continue_input : public continue_receiver { public: @@ -631,18 +684,23 @@ namespace internal { //! The output type of this receiver typedef Output output_type; typedef function_body function_body_type; + typedef continue_input class_type; template< typename Body > - continue_input( graph &g, Body& body ) - : my_graph_ref(g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } + continue_input( graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) ) + : continue_receiver(__TBB_FLOW_GRAPH_PRIORITY_ARG1(/*number_of_predecessors=*/0, priority)) + , my_graph_ref(g) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + { } template< typename Body > - continue_input( graph &g, int number_of_predecessors, Body& body ) - : continue_receiver( number_of_predecessors ), my_graph_ref(g), - my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ), - my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + continue_input( graph &g, int number_of_predecessors, + __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) + ) : continue_receiver( __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, priority) ) + , my_graph_ref(g) + , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { } continue_input( const continue_input& src ) : continue_receiver(src), @@ -678,34 +736,42 @@ namespace internal { virtual broadcast_cache &successors() = 0; - friend class apply_body_task_bypass< continue_input< Output >, continue_msg >; + friend class apply_body_task_bypass< class_type, continue_msg >; //! Applies the body to the provided input task *apply_body_bypass( input_type ) { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE // There is an extra copied needed to capture the // body execution without the try_put tbb::internal::fgt_begin_body( my_body ); output_type v = (*my_body)( continue_msg() ); tbb::internal::fgt_end_body( my_body ); return successors().try_put_task( v ); -#else - return successors().try_put_task( (*my_body)( continue_msg() ) ); -#endif } - //! Spawns a task that applies the body - task *execute( ) __TBB_override { - return (internal::is_graph_active(my_graph_ref)) ? - new ( task::allocate_additional_child_of( *(my_graph_ref.root_task()) ) ) - apply_body_task_bypass< continue_input< Output >, continue_msg >( *this, continue_msg() ) : - NULL; + task* execute() __TBB_override { + if(!internal::is_graph_active(my_graph_ref)) { + return NULL; + } +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(internal::has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + return apply_body_bypass( continue_msg() ); + } + else { + return new ( task::allocate_additional_child_of( *(my_graph_ref.root_task()) ) ) + apply_body_task_bypass< class_type, continue_msg >( + *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(continue_msg(), my_priority) ); + } } graph& graph_reference() __TBB_override { return my_graph_ref; } - }; // continue_input //! Implements methods for both executable and function nodes that puts Output to its successors @@ -717,7 +783,7 @@ namespace internal { typedef Output output_type; typedef typename sender::successor_type successor_type; typedef broadcast_cache broadcast_cache_type; -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION typedef typename sender::built_successors_type built_successors_type; typedef typename sender::successor_list_type successor_list_type; #endif @@ -739,7 +805,7 @@ namespace internal { return true; } -#if TBB_PREVIEW_FLOW_GRAPH_FEATURES +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION built_successors_type &built_successors() __TBB_override { return successors().built_successors(); } @@ -758,7 +824,7 @@ namespace internal { void copy_successors( successor_list_type &v) __TBB_override { successors().copy_successors(v); } -#endif /* TBB_PREVIEW_FLOW_GRAPH_FEATURES */ +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ // for multifunction_node. The function_body that implements // the node will have an input and an output tuple of ports. To put @@ -808,7 +874,7 @@ namespace internal { }; // multifunction_output //composite_node -#if TBB_PREVIEW_FLOW_GRAPH_TRACE && __TBB_FLOW_GRAPH_CPP11_FEATURES +#if __TBB_FLOW_GRAPH_CPP11_FEATURES template void add_nodes_impl(CompositeType*, bool) {} @@ -816,10 +882,7 @@ namespace internal { void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { void *addr = const_cast(&n1); - if(visible) - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, c_node, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_parent_of, addr, tbb::internal::FLOW_NODE ); - else - tbb::internal::itt_relation_add( tbb::internal::ITT_DOMAIN_FLOW, addr, tbb::internal::FLOW_NODE, tbb::internal::__itt_relation_is_child_of, c_node, tbb::internal::FLOW_NODE ); + fgt_alias_port(c_node, addr, visible); add_nodes_impl(c_node, visible, n...); } #endif diff --git a/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h b/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h index 493f76e8..2a1d43e0 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h +++ b/src/tbb/include/tbb/internal/_flow_graph_streaming_node.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_flow_graph_streaming_H diff --git a/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h b/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h index 46755fe0..87a32fe6 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_tagged_buffer_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // a hash table buffer that can expand, and can support as many deletions as diff --git a/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h b/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h index 111810c3..aee40762 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_trace_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _FGT_GRAPH_TRACE_IMPL_H @@ -26,7 +22,18 @@ namespace tbb { namespace internal { -#if TBB_PREVIEW_FLOW_GRAPH_TRACE +#if TBB_USE_THREADING_TOOLS + +static inline void fgt_alias_port(void *node, void *p, bool visible) { + if(visible) + itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE ); + else + itt_relation_add( ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +static inline void fgt_composite ( void *node, void *graph ) { + itt_make_task_group( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE ); +} static inline void fgt_internal_alias_input_port( void *node, void *p, string_index name_index ) { itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); @@ -88,7 +95,10 @@ static inline void fgt_internal_create_output_port( void *node, void *p, string_ template void register_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { // TODO: Make fgt_internal_create_input_port a function template? - fgt_internal_create_input_port( node, port, name_index); + // In C++03 dependent name lookup from the template definition context + // works only for function declarations with external linkage: + // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#561 + fgt_internal_create_input_port(node, static_cast(port), name_index); } template < typename PortsTuple, int N > @@ -239,7 +249,7 @@ static inline void fgt_async_reserve( void *node, void *graph ) { itt_region_begin( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); } -static inline void fgt_async_commit( void *node, void *graph ) { +static inline void fgt_async_commit( void *node, void * /*graph*/) { itt_region_end( ITT_DOMAIN_FLOW, node, FLOW_NODE ); } @@ -251,7 +261,11 @@ static inline void fgt_release_wait( void *graph ) { itt_region_end( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); } -#else // TBB_PREVIEW_FLOW_GRAPH_TRACE +#else // TBB_USE_THREADING_TOOLS + +static inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { } + +static inline void fgt_composite ( void * /*node*/, void * /*graph*/ ) { } static inline void fgt_graph( void * /*g*/ ) { } @@ -296,7 +310,20 @@ static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } static inline void fgt_reserve_wait( void * /*graph*/ ) { } static inline void fgt_release_wait( void * /*graph*/ ) { } -#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +#endif // TBB_USE_THREADING_TOOLS } // namespace internal } // namespace tbb diff --git a/src/tbb/include/tbb/internal/_flow_graph_types_impl.h b/src/tbb/include/tbb/internal/_flow_graph_types_impl.h index 73b5f547..e223dae6 100644 --- a/src/tbb/include/tbb/internal/_flow_graph_types_impl.h +++ b/src/tbb/include/tbb/internal/_flow_graph_types_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__flow_graph_types_impl_H @@ -48,7 +44,7 @@ namespace internal { typedef KHashp KHash; }; -// wrap each element of a tuple in a template, and make a tuple of the result. + // wrap each element of a tuple in a template, and make a tuple of the result. template class PT, typename TypeTuple> struct wrap_tuple_elements; @@ -57,6 +53,19 @@ namespace internal { template class PT, typename KeyTraits, typename TypeTuple> struct wrap_key_tuple_elements; +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT + template class PT, typename... Args> + struct wrap_tuple_elements >{ + typedef typename tbb::flow::tuple... > type; + }; + + template class PT, typename KeyTraits, typename... Args> + struct wrap_key_tuple_elements > { + typedef typename KeyTraits::key_type K; + typedef typename KeyTraits::hash_compare_type KHash; + typedef typename tbb::flow::tuple >... > type; + }; +#else template class PT, typename TypeTuple> struct wrap_tuple_elements<1, PT, TypeTuple> { typedef typename tbb::flow::tuple< @@ -314,6 +323,7 @@ namespace internal { PT > type; }; #endif +#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT */ #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT template< int... S > class sequence {}; diff --git a/src/tbb/include/tbb/internal/_mutex_padding.h b/src/tbb/include/tbb/internal/_mutex_padding.h index 6c1d9b59..09fccd44 100644 --- a/src/tbb/include/tbb/internal/_mutex_padding.h +++ b/src/tbb/include/tbb/internal/_mutex_padding.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mutex_padding_H diff --git a/src/tbb/include/tbb/internal/_node_handle_impl.h b/src/tbb/include/tbb/internal/_node_handle_impl.h new file mode 100644 index 00000000..a910b5fa --- /dev/null +++ b/src/tbb/include/tbb/internal/_node_handle_impl.h @@ -0,0 +1,168 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_node_handle_H +#define __TBB_node_handle_H + +#include "_allocator_traits.h" +#include "../tbb_config.h" + + +namespace tbb { + +// This classes must be declared here for correct friendly relationship +// TODO: Consider creation some internal class to access node_handle private fields without any friendly classes +namespace interface5 { +namespace internal { + template + class split_ordered_list; + template + class concurrent_unordered_base; +} +} + +namespace interface10{ +namespace internal { + template + class concurrent_skip_list; +} +} + +namespace internal { + +template +class node_handle_base { +public: + typedef Allocator allocator_type; +protected: + typedef Node node; + typedef tbb::internal::allocator_traits traits_type; +public: + + node_handle_base() : my_node(NULL), my_allocator() {} + node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), + my_allocator(std::move(nh.my_allocator)) { + nh.my_node = NULL; + } + + bool empty() const { return my_node == NULL; } + explicit operator bool() const { return my_node != NULL; } + + ~node_handle_base() { internal_destroy(); } + + node_handle_base& operator=(node_handle_base&& nh) { + internal_destroy(); + my_node = nh.my_node; + typedef typename traits_type::propagate_on_container_move_assignment pocma_type; + tbb::internal::allocator_move_assignment(my_allocator, nh.my_allocator, pocma_type()); + nh.deactivate(); + return *this; + } + + void swap(node_handle_base& nh) { + std::swap(my_node, nh.my_node); + typedef typename traits_type::propagate_on_container_swap pocs_type; + tbb::internal::allocator_swap(my_allocator, nh.my_allocator, pocs_type()); + } + + allocator_type get_allocator() const { + return my_allocator; + } + +protected: + node_handle_base(node* n) : my_node(n) {} + + void internal_destroy() { + if(my_node) { + traits_type::destroy(my_allocator, my_node->storage()); + typename tbb::internal::allocator_rebind::type node_allocator; + node_allocator.deallocate(my_node, 1); + } + } + + void deactivate() { my_node = NULL; } + + node* my_node; + allocator_type my_allocator; +}; + +// node handle for maps +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key key_type; + typedef typename Value::second_type mapped_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + key_type& key() const { + __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); + return *const_cast(&(this->my_node->value().first)); + } + + mapped_type& mapped() const { + __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); + return this->my_node->value().second; + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + +// node handle for sets +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key value_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + value_type& value() const { + __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); + return *const_cast(&(this->my_node->value())); + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + + +}// namespace internal +}// namespace tbb + +#endif /*__TBB_node_handle_H*/ diff --git a/src/tbb/include/tbb/internal/_range_iterator.h b/src/tbb/include/tbb/internal/_range_iterator.h index 5ebc42e8..733c795f 100644 --- a/src/tbb/include/tbb/internal/_range_iterator.h +++ b/src/tbb/include/tbb/internal/_range_iterator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_range_iterator_H diff --git a/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h b/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h index 6381e2dc..510bde32 100644 --- a/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h +++ b/src/tbb/include/tbb/internal/_tbb_hash_compare_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // must be included outside namespaces. diff --git a/src/tbb/include/tbb/internal/_tbb_strings.h b/src/tbb/include/tbb/internal/_tbb_strings.h index 5c836e64..a5fd3ce5 100644 --- a/src/tbb/include/tbb/internal/_tbb_strings.h +++ b/src/tbb/include/tbb/internal/_tbb_strings.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ TBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, "broadcast_node") @@ -66,6 +62,15 @@ TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") TBB_STRING_RESOURCE(FLOW_OPENCL_NODE, "opencl_node") -// TODO: Drop following string prefix "tbb_" here and in FGA's collector -TBB_STRING_RESOURCE(FGT_ALGORITHM, "tbb_algorithm") -TBB_STRING_RESOURCE(FGT_PARALLEL_FOR, "tbb_parallel_for") +TBB_STRING_RESOURCE(ALGORITHM, "tbb_algorithm") +TBB_STRING_RESOURCE(PARALLEL_FOR, "tbb_parallel_for") +TBB_STRING_RESOURCE(PARALLEL_DO, "tbb_parallel_do") +TBB_STRING_RESOURCE(PARALLEL_INVOKE, "tbb_parallel_invoke") +TBB_STRING_RESOURCE(PARALLEL_REDUCE, "tbb_parallel_reduce") +TBB_STRING_RESOURCE(PARALLEL_SCAN, "tbb_parallel_scan") +TBB_STRING_RESOURCE(PARALLEL_SORT, "tbb_parallel_sort") +TBB_STRING_RESOURCE(CUSTOM_CTX, "tbb_custom") +TBB_STRING_RESOURCE(FLOW_TASKS, "tbb_flow_graph") +TBB_STRING_RESOURCE(PARALLEL_FOR_TASK, "tbb_parallel_for_task") +// TODO: Drop following string prefix "fgt_" here and in FGA's collector +TBB_STRING_RESOURCE(USER_EVENT, "fgt_user_event") diff --git a/src/tbb/include/tbb/internal/_tbb_trace_impl.h b/src/tbb/include/tbb/internal/_tbb_trace_impl.h index e7eb5784..e89ab232 100644 --- a/src/tbb/include/tbb/internal/_tbb_trace_impl.h +++ b/src/tbb/include/tbb/internal/_tbb_trace_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _FGT_TBB_TRACE_IMPL_H @@ -27,18 +23,17 @@ namespace tbb { namespace internal { #if TBB_PREVIEW_ALGORITHM_TRACE - static inline void fgt_algorithm( string_index t, void *algorithm, void *parent ) { - itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, parent, FGT_ALGORITHM, t ); + itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, parent, ALGORITHM, t ); } static inline void fgt_begin_algorithm( string_index t, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, algorithm, FGT_ALGORITHM, NULL, FLOW_NULL, t ); + itt_task_begin( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, NULL, FLOW_NULL, t ); } static inline void fgt_end_algorithm( void * ) { itt_task_end( ITT_DOMAIN_FLOW ); } static inline void fgt_alg_begin_body( string_index t, void *body, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, FGT_ALGORITHM, t ); + itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, ALGORITHM, t ); } static inline void fgt_alg_end_body( void * ) { itt_task_end( ITT_DOMAIN_FLOW ); diff --git a/src/tbb/include/tbb/internal/_tbb_windef.h b/src/tbb/include/tbb/internal/_tbb_windef.h index e798dee4..1268ba27 100644 --- a/src/tbb/include/tbb/internal/_tbb_windef.h +++ b/src/tbb/include/tbb/internal/_tbb_windef.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,17 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_windef_H #error Do not #include this internal file directly; use public TBB headers instead. #endif /* __TBB_tbb_windef_H */ -// Check that the target Windows version has all API calls requried for TBB. +// Check that the target Windows version has all API calls required for TBB. // Do not increase the version in condition beyond 0x0500 without prior discussion! #if defined(_WIN32_WINNT) && _WIN32_WINNT<0x0501 #error TBB is unable to run on old Windows versions; _WIN32_WINNT must be 0x0501 or greater. diff --git a/src/tbb/include/tbb/internal/_template_helpers.h b/src/tbb/include/tbb/internal/_template_helpers.h index 1e0abbe8..87c3efbe 100644 --- a/src/tbb/include/tbb/internal/_template_helpers.h +++ b/src/tbb/include/tbb/internal/_template_helpers.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_template_helpers_H @@ -23,6 +19,14 @@ #include #include +#include "../tbb_config.h" +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT +#include +#endif +#if __TBB_CPP11_PRESENT +#include +#include // allocator_traits +#endif namespace tbb { namespace internal { @@ -60,11 +64,41 @@ template struct is_same_type { static const bool value = template struct is_ref { static const bool value = false; }; template struct is_ref { static const bool value = true; }; +//! Partial support for std::is_integral +template struct is_integral_impl { static const bool value = false; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#if __TBB_CPP11_PRESENT +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#endif +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; + +template +struct is_integral : is_integral_impl::type> {}; + #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT //! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) template struct void_t { typedef void type; }; #endif +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502 +template class... Checks> +struct supports_impl { typedef std::false_type type; }; +template class... Checks> +struct supports_impl...>::type, Checks...> { typedef std::true_type type; }; + +template class... Checks> +using supports = typename supports_impl::type; + +#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ + #if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT //! Allows to store a function parameter pack as a variable and later pass it to another function @@ -153,6 +187,81 @@ stored_pack save_pack( Types&&... types ) { } #endif /* __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ + +#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT + +using std::index_sequence; +using std::make_index_sequence; + +#elif __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT + +template class index_sequence {}; + +template +struct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {}; + +template +struct make_index_sequence_impl <0, S...> { + using type = index_sequence; +}; + +template +using make_index_sequence = typename tbb::internal::make_index_sequence_impl::type; + +#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */ + +#if __TBB_CPP11_PRESENT + +template< typename Iter > +using iterator_value_t = typename std::iterator_traits::value_type; + +template< typename Iter > +using iterator_key_t = typename std::remove_const::first_type>::type; + +template< typename Iter > +using iterator_mapped_t = typename iterator_value_t::second_type; + +template< typename A > using value_type = typename A::value_type; +template< typename A > using alloc_ptr_t = typename std::allocator_traits::pointer; +template< typename A > using has_allocate = decltype(std::declval&>() = std::declval().allocate(0)); +template< typename A > using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0)); + +// value_type should be checked first because it can be used in other checks (via allocator_traits) +template< typename T > +using is_allocator = supports; + +#if __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT + +template< typename T > +static constexpr bool is_allocator_v = is_allocator::value; + +#endif /*__TBB_CPP14_VARIABLE_TEMPLATES */ + +template< std::size_t N, typename... Args > +struct pack_element { + using type = void; +}; + +template< std::size_t N, typename T, typename... Args > +struct pack_element { + using type = typename pack_element::type; +}; + +template< typename T, typename... Args > +struct pack_element<0, T, Args...> { + using type = T; +}; + +template< std::size_t N, typename... Args > +using pack_element_t = typename pack_element::type; + +template using is_transparent = typename Comp::is_transparent; + +template +using has_is_transparent = supports; + +#endif /* __TBB_CPP11_PRESENT */ + } } // namespace internal, namespace tbb #endif /* __TBB_template_helpers_H */ diff --git a/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h b/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h index ef5f9223..a03e463f 100644 --- a/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h +++ b/src/tbb/include/tbb/internal/_x86_eliding_mutex_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__x86_eliding_mutex_impl_H diff --git a/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h b/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h index b08c2331..0003abac 100644 --- a/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h +++ b/src/tbb/include/tbb/internal/_x86_rtm_rw_mutex_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB__x86_rtm_rw_mutex_impl_H @@ -88,11 +84,11 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { static x86_rtm_rw_mutex* internal_get_mutex( const spin_rw_mutex::scoped_lock& lock ) { - return static_cast( lock.internal_get_mutex() ); + return static_cast( lock.mutex ); } static void internal_set_mutex( spin_rw_mutex::scoped_lock& lock, spin_rw_mutex* mtx ) { - lock.internal_set_mutex( mtx ); + lock.mutex = mtx; } //! @endcond public: @@ -171,7 +167,8 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { bool upgrade_to_writer() { x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_reader || transaction_state==RTM_real_reader, "Invalid state for upgrade" ); + if (transaction_state == RTM_transacting_writer || transaction_state == RTM_real_writer) + return true; // Already a writer return mutex->internal_upgrade(*this); } @@ -180,7 +177,8 @@ class x86_rtm_rw_mutex: private spin_rw_mutex { bool downgrade_to_reader() { x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state==RTM_transacting_writer || transaction_state==RTM_real_writer, "Invalid state for downgrade" ); + if (transaction_state == RTM_transacting_reader || transaction_state == RTM_real_reader) + return true; // Already a reader return mutex->internal_downgrade(*this); } diff --git a/src/tbb/include/tbb/iterators.h b/src/tbb/include/tbb/iterators.h new file mode 100644 index 00000000..7a3c9273 --- /dev/null +++ b/src/tbb/include/tbb/iterators.h @@ -0,0 +1,326 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_iterators_H +#define __TBB_iterators_H + +#include +#include + +#include "tbb_config.h" +#include "tbb_stddef.h" + +#if __TBB_CPP11_PRESENT + +#include + +namespace tbb { + +template +class counting_iterator { + __TBB_STATIC_ASSERT(std::numeric_limits::is_integer, "Cannot instantiate counting_iterator with a non-integer type"); +public: + typedef typename std::make_signed::type difference_type; + typedef IntType value_type; + typedef const IntType* pointer; + typedef const IntType& reference; + typedef std::random_access_iterator_tag iterator_category; + + counting_iterator() : my_counter() {} + explicit counting_iterator(IntType init) : my_counter(init) {} + + reference operator*() const { return my_counter; } + value_type operator[](difference_type i) const { return *(*this + i); } + + difference_type operator-(const counting_iterator& it) const { return my_counter - it.my_counter; } + + counting_iterator& operator+=(difference_type forward) { my_counter += forward; return *this; } + counting_iterator& operator-=(difference_type backward) { return *this += -backward; } + counting_iterator& operator++() { return *this += 1; } + counting_iterator& operator--() { return *this -= 1; } + + counting_iterator operator++(int) { + counting_iterator it(*this); + ++(*this); + return it; + } + counting_iterator operator--(int) { + counting_iterator it(*this); + --(*this); + return it; + } + + counting_iterator operator-(difference_type backward) const { return counting_iterator(my_counter - backward); } + counting_iterator operator+(difference_type forward) const { return counting_iterator(my_counter + forward); } + friend counting_iterator operator+(difference_type forward, const counting_iterator it) { return it + forward; } + + bool operator==(const counting_iterator& it) const { return *this - it == 0; } + bool operator!=(const counting_iterator& it) const { return !(*this == it); } + bool operator<(const counting_iterator& it) const {return *this - it < 0; } + bool operator>(const counting_iterator& it) const { return it < *this; } + bool operator<=(const counting_iterator& it) const { return !(*this > it); } + bool operator>=(const counting_iterator& it) const { return !(*this < it); } + +private: + IntType my_counter; +}; +} //namespace tbb + + +#include + +#include "internal/_template_helpers.h" // index_sequence, make_index_sequence + +namespace tbb { +namespace internal { + +template +struct tuple_util { + template + static void increment(TupleType& it, DifferenceType forward) { + std::get(it) += forward; + tuple_util::increment(it, forward); + } + template + static bool check_sync(const TupleType& it1, const TupleType& it2, DifferenceType val) { + if(std::get(it1) - std::get(it2) != val) + return false; + return tuple_util::check_sync(it1, it2, val); + } +}; + +template<> +struct tuple_util<0> { + template + static void increment(TupleType&, DifferenceType) {} + template + static bool check_sync(const TupleType&, const TupleType&, DifferenceType) { return true;} +}; + +template +struct make_references { + template + TupleReturnType operator()(const TupleType& t, tbb::internal::index_sequence) { + return std::tie( *std::get(t)... ); + } +}; + +// A simple wrapper over a tuple of references. +// The class is designed to hold a temporary tuple of reference +// after dereferencing a zip_iterator; in particular, it is needed +// to swap these rvalue tuples. Any other usage is not supported. +template +struct tuplewrapper : public std::tuple::value, T&&>::type...> { + // In the context of this class, T is a reference, so T&& is a "forwarding reference" + typedef std::tuple base_type; + // Construct from the result of std::tie + tuplewrapper(const base_type& in) : base_type(in) {} +#if __INTEL_COMPILER + // ICC cannot generate copy ctor & assignment + tuplewrapper(const tuplewrapper& rhs) : base_type(rhs) {} + tuplewrapper& operator=(const tuplewrapper& rhs) { + *this = base_type(rhs); + return *this; + } +#endif + // Assign any tuple convertible to std::tuple: *it = a_tuple; + template + tuplewrapper& operator=(const std::tuple& other) { + base_type::operator=(other); + return *this; + } +#if _LIBCPP_VERSION + // (Necessary for libc++ tuples) Convert to a tuple of values: v = *it; + operator std::tuple::type...>() { return base_type(*this); } +#endif + // Swap rvalue tuples: swap(*it1,*it2); + friend void swap(tuplewrapper&& a, tuplewrapper&& b) { + std::swap(a,b); + } +}; + +} //namespace internal + +template +class zip_iterator { + __TBB_STATIC_ASSERT(sizeof...(Types), "Cannot instantiate zip_iterator with empty template parameter pack"); + static const std::size_t num_types = sizeof...(Types); + typedef std::tuple it_types; +public: + typedef typename std::make_signed::type difference_type; + typedef std::tuple::value_type...> value_type; +#if __INTEL_COMPILER && __INTEL_COMPILER < 1800 && _MSC_VER + typedef std::tuple::reference...> reference; +#else + typedef tbb::internal::tuplewrapper::reference...> reference; +#endif + typedef std::tuple::pointer...> pointer; + typedef std::random_access_iterator_tag iterator_category; + + zip_iterator() : my_it() {} + explicit zip_iterator(Types... args) : my_it(std::make_tuple(args...)) {} + zip_iterator(const zip_iterator& input) : my_it(input.my_it) {} + zip_iterator& operator=(const zip_iterator& input) { + my_it = input.my_it; + return *this; + } + + reference operator*() const { + return tbb::internal::make_references()(my_it, tbb::internal::make_index_sequence()); + } + reference operator[](difference_type i) const { return *(*this + i); } + + difference_type operator-(const zip_iterator& it) const { + __TBB_ASSERT(internal::tuple_util::check_sync(my_it, it.my_it, std::get<0>(my_it) - std::get<0>(it.my_it)), + "Components of zip_iterator are not synchronous"); + return std::get<0>(my_it) - std::get<0>(it.my_it); + } + + zip_iterator& operator+=(difference_type forward) { + internal::tuple_util::increment(my_it, forward); + return *this; + } + zip_iterator& operator-=(difference_type backward) { return *this += -backward; } + zip_iterator& operator++() { return *this += 1; } + zip_iterator& operator--() { return *this -= 1; } + + zip_iterator operator++(int) { + zip_iterator it(*this); + ++(*this); + return it; + } + zip_iterator operator--(int) { + zip_iterator it(*this); + --(*this); + return it; + } + + zip_iterator operator-(difference_type backward) const { + zip_iterator it(*this); + return it -= backward; + } + zip_iterator operator+(difference_type forward) const { + zip_iterator it(*this); + return it += forward; + } + friend zip_iterator operator+(difference_type forward, const zip_iterator& it) { return it + forward; } + + bool operator==(const zip_iterator& it) const { + return *this - it == 0; + } + it_types base() const { return my_it; } + + bool operator!=(const zip_iterator& it) const { return !(*this == it); } + bool operator<(const zip_iterator& it) const { return *this - it < 0; } + bool operator>(const zip_iterator& it) const { return it < *this; } + bool operator<=(const zip_iterator& it) const { return !(*this > it); } + bool operator>=(const zip_iterator& it) const { return !(*this < it); } +private: + it_types my_it; +}; + +template +zip_iterator make_zip_iterator(T... args) { return zip_iterator(args...); } + +template +class transform_iterator { +public: + typedef typename std::iterator_traits::value_type value_type; + typedef typename std::iterator_traits::difference_type difference_type; +#if __TBB_CPP17_INVOKE_RESULT_PRESENT + typedef typename std::invoke_result::reference>::type reference; +#else + typedef typename std::result_of::reference)>::type reference; +#endif + typedef typename std::iterator_traits::pointer pointer; + typedef typename std::random_access_iterator_tag iterator_category; + + transform_iterator(Iter it, UnaryFunc unary_func) : my_it(it), my_unary_func(unary_func) { + __TBB_STATIC_ASSERT((std::is_same::iterator_category, + std::random_access_iterator_tag>::value), "Random access iterator required."); + } + transform_iterator(const transform_iterator& input) : my_it(input.my_it), my_unary_func(input.my_unary_func) { } + transform_iterator& operator=(const transform_iterator& input) { + my_it = input.my_it; + return *this; + } + reference operator*() const { + return my_unary_func(*my_it); + } + reference operator[](difference_type i) const { + return *(*this + i); + } + transform_iterator& operator++() { + ++my_it; + return *this; + } + transform_iterator& operator--() { + --my_it; + return *this; + } + transform_iterator operator++(int) { + transform_iterator it(*this); + ++(*this); + return it; + } + transform_iterator operator--(int) { + transform_iterator it(*this); + --(*this); + return it; + } + transform_iterator operator+(difference_type forward) const { + return { my_it + forward, my_unary_func }; + } + transform_iterator operator-(difference_type backward) const { + return { my_it - backward, my_unary_func }; + } + transform_iterator& operator+=(difference_type forward) { + my_it += forward; + return *this; + } + transform_iterator& operator-=(difference_type backward) { + my_it -= backward; + return *this; + } + friend transform_iterator operator+(difference_type forward, const transform_iterator& it) { + return it + forward; + } + difference_type operator-(const transform_iterator& it) const { + return my_it - it.my_it; + } + bool operator==(const transform_iterator& it) const { return *this - it == 0; } + bool operator!=(const transform_iterator& it) const { return !(*this == it); } + bool operator<(const transform_iterator& it) const { return *this - it < 0; } + bool operator>(const transform_iterator& it) const { return it < *this; } + bool operator<=(const transform_iterator& it) const { return !(*this > it); } + bool operator>=(const transform_iterator& it) const { return !(*this < it); } + + Iter base() const { return my_it; } +private: + Iter my_it; + const UnaryFunc my_unary_func; +}; + +template +transform_iterator make_transform_iterator(Iter it, UnaryFunc unary_func) { + return transform_iterator(it, unary_func); +} + +} //namespace tbb + +#endif //__TBB_CPP11_PRESENT + +#endif /* __TBB_iterators_H */ diff --git a/src/tbb/include/tbb/machine/gcc_arm.h b/src/tbb/include/tbb/machine/gcc_arm.h new file mode 100644 index 00000000..40118e87 --- /dev/null +++ b/src/tbb/include/tbb/machine/gcc_arm.h @@ -0,0 +1,216 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + Platform isolation layer for the ARMv7-a architecture. +*/ + +#ifndef __TBB_machine_H +#error Do not include this file directly; include tbb_machine.h instead +#endif + +#if __ARM_ARCH_7A__ + +#include +#include + +#define __TBB_WORDSIZE 4 + +// Traditionally ARM is little-endian. +// Note that, since only the layout of aligned 32-bit words is of interest, +// any apparent PDP-endianness of 32-bit words at half-word alignment or +// any little-endian ordering of big-endian 32-bit words in 64-bit quantities +// may be disregarded for this setting. +#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG +#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE +#elif defined(__BYTE_ORDER__) + #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED +#else + #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT +#endif + + +#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") +#define __TBB_full_memory_fence() __asm__ __volatile__("dmb ish": : :"memory") +#define __TBB_control_consistency_helper() __TBB_full_memory_fence() +#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() +#define __TBB_release_consistency_helper() __TBB_full_memory_fence() + +//-------------------------------------------------- +// Compare and swap +//-------------------------------------------------- + +/** + * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr + * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand + * @param value value to assign *ptr to if *ptr==comparand + * @param comparand value to compare with *ptr + * @return value originally in memory at ptr, regardless of success +*/ +static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ) +{ + int32_t oldval, res; + + __TBB_full_memory_fence(); + + do { + __asm__ __volatile__( + "ldrex %1, [%3]\n" + "mov %0, #0\n" + "cmp %1, %4\n" + "it eq\n" + "strexeq %0, %5, [%3]\n" + : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int32_t*)ptr) + : "r" ((volatile int32_t *)ptr), "Ir" (comparand), "r" (value) + : "cc"); + } while (res); + + __TBB_full_memory_fence(); + + return oldval; +} + +/** + * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr + * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand + * @param value value to assign *ptr to if *ptr==comparand + * @param comparand value to compare with *ptr + * @return value originally in memory at ptr, regardless of success + */ +static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ) +{ + int64_t oldval; + int32_t res; + + __TBB_full_memory_fence(); + + do { + __asm__ __volatile__( + "mov %0, #0\n" + "ldrexd %1, %H1, [%3]\n" + "cmp %1, %4\n" + "it eq\n" + "cmpeq %H1, %H4\n" + "it eq\n" + "strexdeq %0, %5, %H5, [%3]" + : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int64_t*)ptr) + : "r" ((volatile int64_t *)ptr), "r" (comparand), "r" (value) + : "cc"); + } while (res); + + __TBB_full_memory_fence(); + + return oldval; +} + +static inline int32_t __TBB_machine_fetchadd4(volatile void* ptr, int32_t addend) +{ + unsigned long tmp; + int32_t result, tmp2; + + __TBB_full_memory_fence(); + + __asm__ __volatile__( +"1: ldrex %0, [%4]\n" +" add %3, %0, %5\n" +" strex %1, %3, [%4]\n" +" cmp %1, #0\n" +" bne 1b\n" + : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int32_t*)ptr), "=&r"(tmp2) + : "r" ((volatile int32_t *)ptr), "Ir" (addend) + : "cc"); + + __TBB_full_memory_fence(); + + return result; +} + +static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) +{ + unsigned long tmp; + int64_t result, tmp2; + + __TBB_full_memory_fence(); + + __asm__ __volatile__( +"1: ldrexd %0, %H0, [%4]\n" +" adds %3, %0, %5\n" +" adc %H3, %H0, %H5\n" +" strexd %1, %3, %H3, [%4]\n" +" cmp %1, #0\n" +" bne 1b" + : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int64_t*)ptr), "=&r"(tmp2) + : "r" ((volatile int64_t *)ptr), "r" (addend) + : "cc"); + + + __TBB_full_memory_fence(); + + return result; +} + +namespace tbb { +namespace internal { + template + struct machine_load_store_relaxed { + static inline T load ( const volatile T& location ) { + const T value = location; + + /* + * An extra memory barrier is required for errata #761319 + * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a + */ + __TBB_acquire_consistency_helper(); + return value; + } + + static inline void store ( volatile T& location, T value ) { + location = value; + } + }; +}} // namespaces internal, tbb + +// Machine specific atomic operations + +#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) +#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) + +// Use generics for some things +#define __TBB_USE_GENERIC_PART_WORD_CAS 1 +#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 +#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 +#define __TBB_USE_GENERIC_FETCH_STORE 1 +#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 +#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 +#elif defined __aarch64__ +// Generic gcc implementations are fine for ARMv8-a except __TBB_PAUSE. +#include "gcc_generic.h" +#else +#error compilation requires an ARMv7-a or ARMv8-a architecture. +#endif // __ARM_ARCH_7A__ + +inline void __TBB_machine_pause (int32_t delay) +{ + while(delay>0) + { + __asm__ __volatile__("yield" ::: "memory"); + delay--; + } +} +#define __TBB_Pause(V) __TBB_machine_pause(V) diff --git a/src/tbb/include/tbb/machine/gcc_generic.h b/src/tbb/include/tbb/machine/gcc_generic.h index 5fc2a901..cbf8d993 100644 --- a/src/tbb/include/tbb/machine/gcc_generic.h +++ b/src/tbb/include/tbb/machine/gcc_generic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_generic_H) @@ -106,23 +102,17 @@ __TBB_MACHINE_DEFINE_ATOMICS(8,int64_t) #undef __TBB_MACHINE_DEFINE_ATOMICS -namespace tbb{ namespace internal { namespace gcc_builtins { - inline int clz(unsigned int x){ return __builtin_clz(x);}; - inline int clz(unsigned long int x){ return __builtin_clzl(x);}; - inline int clz(unsigned long long int x){ return __builtin_clzll(x);}; -}}} -//gcc __builtin_clz builtin count _number_ of leading zeroes -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - return sizeof(x)*8 - tbb::internal::gcc_builtins::clz(x) -1 ; -} - - typedef unsigned char __TBB_Flag; typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; #if __TBB_GCC_VERSION < 40700 // Use __sync_* builtins +// Use generic machine_load_store functions if there are no builtin atomics +#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 +#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 + static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { __sync_fetch_and_or(reinterpret_cast(ptr),addend); } @@ -158,6 +148,59 @@ inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { __atomic_clear(&flag,__ATOMIC_RELEASE); } +namespace tbb { namespace internal { + +/** GCC atomic operation intrinsics might miss compiler fence. + Adding it after load-with-acquire, before store-with-release, and + on both sides of sequentially consistent operations is sufficient for correctness. **/ + +template +inline T __TBB_machine_atomic_load( const volatile T& location) { + if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); + T value = __atomic_load_n(&location, MemOrder); + if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); + return value; +} + +template +inline void __TBB_machine_atomic_store( volatile T& location, T value) { + if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); + __atomic_store_n(&location, value, MemOrder); + if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); +} + +template +struct machine_load_store { + static T load_with_acquire ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static void store_with_release ( volatile T &location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +template +struct machine_load_store_relaxed { + static inline T load ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static inline void store ( volatile T& location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +template +struct machine_load_store_seq_cst { + static T load ( const volatile T& location ) { + return __TBB_machine_atomic_load(location); + } + static void store ( volatile T &location, T value ) { + __TBB_machine_atomic_store(location, value); + } +}; + +}} // namespace tbb::internal + #endif // __TBB_GCC_VERSION < 40700 // Machine specific atomic operations @@ -167,18 +210,24 @@ inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { #define __TBB_TryLockByte __TBB_machine_try_lock_byte #define __TBB_UnlockByte __TBB_machine_unlock_byte -// Definition of other functions -#define __TBB_Log2(V) __TBB_machine_lg(V) +// __builtin_clz counts the number of leading zeroes +namespace tbb{ namespace internal { namespace gcc_builtins { + inline int clz(unsigned int x){ return __builtin_clz(x); } + inline int clz(unsigned long int x){ return __builtin_clzl(x); } + inline int clz(unsigned long long int x){ return __builtin_clzll(x); } +}}} +// logarithm is the index of the most significant non-zero bit +static inline intptr_t __TBB_machine_lg( uintptr_t x ) { + // If P is a power of 2 and x static inline intptr_t __TBB_machine_lg( T x ) { __TBB_ASSERT(x>0, "The logarithm of a non-positive value is undefined."); @@ -33,6 +29,7 @@ static inline intptr_t __TBB_machine_lg( T x ) { return j; } #define __TBB_Log2(V) __TBB_machine_lg(V) +#endif /* !__TBB_Log2 */ #ifndef __TBB_Pause //TODO: check if raising a ratio of pause instructions to loop control instructions diff --git a/src/tbb/include/tbb/machine/gcc_itsx.h b/src/tbb/include/tbb/machine/gcc_itsx.h index caa35441..5bd400e5 100644 --- a/src/tbb/include/tbb/machine/gcc_itsx.h +++ b/src/tbb/include/tbb/machine/gcc_itsx.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_itsx_H) diff --git a/src/tbb/include/tbb/machine/ibm_aix51.h b/src/tbb/include/tbb/machine/ibm_aix51.h index a905b4e1..14ba4d9f 100644 --- a/src/tbb/include/tbb/machine/ibm_aix51.h +++ b/src/tbb/include/tbb/machine/ibm_aix51.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // TODO: revise by comparing with mac_ppc.h diff --git a/src/tbb/include/tbb/machine/icc_generic.h b/src/tbb/include/tbb/machine/icc_generic.h index 04863000..00af78a7 100644 --- a/src/tbb/include/tbb/machine/icc_generic.h +++ b/src/tbb/include/tbb/machine/icc_generic.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_icc_generic_H) @@ -23,7 +19,7 @@ #endif #if ! __TBB_ICC_BUILTIN_ATOMICS_PRESENT - #error "Intel C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" + #error "Intel(R) C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" #endif #define __TBB_machine_icc_generic_H diff --git a/src/tbb/include/tbb/machine/linux_common.h b/src/tbb/include/tbb/machine/linux_common.h index 4d2d355b..6f93828d 100644 --- a/src/tbb/include/tbb/machine/linux_common.h +++ b/src/tbb/include/tbb/machine/linux_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_H @@ -30,19 +26,44 @@ #include #if defined(SYS_futex) +/* This header file is included for Linux and some other systems that may support futexes.*/ #define __TBB_USE_FUTEX 1 + +#if defined(__has_include) +#define __TBB_has_include __has_include +#else +#define __TBB_has_include(x) 0 +#endif + +/* +If available, use typical headers where futex API is defined. While Linux and OpenBSD +are known to provide such headers, other systems might have them as well. +*/ +#if defined(__linux__) || __TBB_has_include() +#include +#elif defined(__OpenBSD__) || __TBB_has_include() +#include +#endif + #include #include -// Unfortunately, some versions of Linux do not have a header that defines FUTEX_WAIT and FUTEX_WAKE. -#ifdef FUTEX_WAIT +/* +Some systems might not define the macros or use different names. In such case we expect +the actual parameter values to match Linux: 0 for wait, 1 for wake. +*/ +#if defined(FUTEX_WAIT_PRIVATE) +#define __TBB_FUTEX_WAIT FUTEX_WAIT_PRIVATE +#elif defined(FUTEX_WAIT) #define __TBB_FUTEX_WAIT FUTEX_WAIT #else #define __TBB_FUTEX_WAIT 0 #endif -#ifdef FUTEX_WAKE +#if defined(FUTEX_WAKE_PRIVATE) +#define __TBB_FUTEX_WAKE FUTEX_WAKE_PRIVATE +#elif defined(FUTEX_WAKE) #define __TBB_FUTEX_WAKE FUTEX_WAKE #else #define __TBB_FUTEX_WAKE 1 diff --git a/src/tbb/include/tbb/machine/linux_ia32.h b/src/tbb/include/tbb/machine/linux_ia32.h index 932d343d..6e2543c7 100644 --- a/src/tbb/include/tbb/machine/linux_ia32.h +++ b/src/tbb/include/tbb/machine/linux_ia32.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia32_H) diff --git a/src/tbb/include/tbb/machine/linux_ia64.h b/src/tbb/include/tbb/machine/linux_ia64.h index f4772288..be9676e1 100644 --- a/src/tbb/include/tbb/machine/linux_ia64.h +++ b/src/tbb/include/tbb/machine/linux_ia64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia64_H) diff --git a/src/tbb/include/tbb/machine/linux_intel64.h b/src/tbb/include/tbb/machine/linux_intel64.h index 02153c2a..d0b15bed 100644 --- a/src/tbb/include/tbb/machine/linux_intel64.h +++ b/src/tbb/include/tbb/machine/linux_intel64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_intel64_H) diff --git a/src/tbb/include/tbb/machine/mac_ppc.h b/src/tbb/include/tbb/machine/mac_ppc.h index 13f387b4..0c07cc99 100644 --- a/src/tbb/include/tbb/machine/mac_ppc.h +++ b/src/tbb/include/tbb/machine/mac_ppc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_power_H) diff --git a/src/tbb/include/tbb/machine/macos_common.h b/src/tbb/include/tbb/machine/macos_common.h index 119ad979..c63536fa 100644 --- a/src/tbb/include/tbb/machine/macos_common.h +++ b/src/tbb/include/tbb/machine/macos_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_macos_common_H) diff --git a/src/tbb/include/tbb/machine/mic_common.h b/src/tbb/include/tbb/machine/mic_common.h index 8765d39f..afe08331 100644 --- a/src/tbb/include/tbb/machine/mic_common.h +++ b/src/tbb/include/tbb/machine/mic_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mic_common_H diff --git a/src/tbb/include/tbb/machine/msvc_armv7.h b/src/tbb/include/tbb/machine/msvc_armv7.h index 40d22020..d38ceb9c 100644 --- a/src/tbb/include/tbb/machine/msvc_armv7.h +++ b/src/tbb/include/tbb/machine/msvc_armv7.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_msvc_armv7_H) diff --git a/src/tbb/include/tbb/machine/msvc_ia32_common.h b/src/tbb/include/tbb/machine/msvc_ia32_common.h index 8b4814bc..541e1fa9 100644 --- a/src/tbb/include/tbb/machine/msvc_ia32_common.h +++ b/src/tbb/include/tbb/machine/msvc_ia32_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_msvc_ia32_common_H) @@ -49,7 +45,7 @@ }}} #endif -#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) +#if __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT // S is the operand size in bytes, B is the suffix for intrinsics for that size #define __TBB_MACHINE_DEFINE_ATOMICS(S,B,T,U) \ __pragma(intrinsic( _InterlockedCompareExchange##B )) \ @@ -75,8 +71,7 @@ #endif #undef __TBB_MACHINE_DEFINE_ATOMICS - #define __TBB_ATOMIC_PRIMITIVES_DEFINED -#endif /*_MSC_VER>=1600*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ #if _MSC_VER>=1300 || __INTEL_COMPILER>=1100 #pragma intrinsic(_ReadWriteBarrier) diff --git a/src/tbb/include/tbb/machine/sunos_sparc.h b/src/tbb/include/tbb/machine/sunos_sparc.h index 9119f402..18e68c34 100644 --- a/src/tbb/include/tbb/machine/sunos_sparc.h +++ b/src/tbb/include/tbb/machine/sunos_sparc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ diff --git a/src/tbb/include/tbb/machine/windows_api.h b/src/tbb/include/tbb/machine/windows_api.h index d362abc5..abf0e182 100644 --- a/src/tbb/include/tbb/machine/windows_api.h +++ b/src/tbb/include/tbb/machine/windows_api.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_windows_api_H diff --git a/src/tbb/include/tbb/machine/windows_ia32.h b/src/tbb/include/tbb/machine/windows_ia32.h index 8db0d2b8..86550a7e 100644 --- a/src/tbb/include/tbb/machine/windows_ia32.h +++ b/src/tbb/include/tbb/machine/windows_ia32.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_ia32_H) @@ -43,7 +39,7 @@ extern "C" { __int64 __TBB_EXPORTED_FUNC __TBB_machine_load8 (const volatile void *ptr); } -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED +#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT #define __TBB_MACHINE_DEFINE_ATOMICS(S,T,U,A,C) \ static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ @@ -93,7 +89,7 @@ __TBB_MACHINE_DEFINE_ATOMICS(4, ptrdiff_t, ptrdiff_t, eax, ecx) #undef __TBB_MACHINE_DEFINE_ATOMICS -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ //TODO: Check if it possible and profitable for IA-32 architecture on (Linux and Windows) //to use of 64-bit load/store via floating point registers together with full fence diff --git a/src/tbb/include/tbb/machine/windows_intel64.h b/src/tbb/include/tbb/machine/windows_intel64.h index 86abd6ad..6b733d20 100644 --- a/src/tbb/include/tbb/machine/windows_intel64.h +++ b/src/tbb/include/tbb/machine/windows_intel64.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_intel64_H) @@ -29,7 +25,7 @@ #include "msvc_ia32_common.h" -#ifndef __TBB_ATOMIC_PRIMITIVES_DEFINED +#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT #include #pragma intrinsic(_InterlockedCompareExchange,_InterlockedExchangeAdd,_InterlockedExchange) @@ -66,7 +62,7 @@ inline __int64 __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ) { return _InterlockedExchange64( (__int64*)ptr, value ); } -#endif /*__TBB_ATOMIC_PRIMITIVES_DEFINED*/ +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ #define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 #define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 diff --git a/src/tbb/include/tbb/memory_pool.h b/src/tbb/include/tbb/memory_pool.h index b3bba397..2037d634 100644 --- a/src/tbb/include/tbb/memory_pool.h +++ b/src/tbb/include/tbb/memory_pool.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_memory_pool_H diff --git a/src/tbb/include/tbb/mutex.h b/src/tbb/include/tbb/mutex.h index e40b4cd0..5f461f3f 100644 --- a/src/tbb/include/tbb/mutex.h +++ b/src/tbb/include/tbb/mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_mutex_H diff --git a/src/tbb/include/tbb/null_mutex.h b/src/tbb/include/tbb/null_mutex.h index 85c660e3..b9c382dd 100644 --- a/src/tbb/include/tbb/null_mutex.h +++ b/src/tbb/include/tbb/null_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_null_mutex_H diff --git a/src/tbb/include/tbb/null_rw_mutex.h b/src/tbb/include/tbb/null_rw_mutex.h index fa0c8035..d2fd33ca 100644 --- a/src/tbb/include/tbb/null_rw_mutex.h +++ b/src/tbb/include/tbb/null_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_null_rw_mutex_H diff --git a/src/tbb/include/tbb/parallel_do.h b/src/tbb/include/tbb/parallel_do.h index 15275682..d3f2075d 100644 --- a/src/tbb/include/tbb/parallel_do.h +++ b/src/tbb/include/tbb/parallel_do.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_do_H @@ -500,7 +496,7 @@ void parallel_do( Iterator first, Iterator last, const Body& body ) if ( first == last ) return; #if __TBB_TASK_GROUP_CONTEXT - task_group_context context; + task_group_context context(internal::PARALLEL_DO); #endif interface9::internal::select_parallel_do( first, last, body, &Body::operator() #if __TBB_TASK_GROUP_CONTEXT diff --git a/src/tbb/include/tbb/parallel_for.h b/src/tbb/include/tbb/parallel_for.h index 19549e19..a682067a 100644 --- a/src/tbb/include/tbb/parallel_for.h +++ b/src/tbb/include/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_for_H @@ -58,7 +54,7 @@ namespace internal { my_body(body), my_partition(partitioner) { - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, NULL); + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, NULL); } //! Splitting constructor used to generate children. /** parent_ becomes left child. Newly constructed object is right child. */ @@ -68,7 +64,7 @@ namespace internal { my_partition(parent_.my_partition, split_obj) { my_partition.set_affinity(*this); - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, (void *)&parent_); + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); } //! Construct right child from the given range as response to the demand. /** parent_ remains left child. Newly constructed object is right child. */ @@ -79,7 +75,7 @@ namespace internal { { my_partition.set_affinity(*this); my_partition.align_depth( d ); - tbb::internal::fgt_algorithm(tbb::internal::FGT_PARALLEL_FOR, this, (void *)&parent_); + tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); } static void run( const Range& range, const Body& body, Partitioner& partitioner ) { if( !range.empty() ) { @@ -88,13 +84,13 @@ namespace internal { #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; + task_group_context context(PARALLEL_FOR); start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ // REGION BEGIN - fgt_begin_algorithm( tbb::internal::FGT_PARALLEL_FOR, (void*)&a ); + fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&a ); + fgt_end_algorithm( (void*)&context ); // REGION END } } @@ -103,16 +99,16 @@ namespace internal { if( !range.empty() ) { start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); // REGION BEGIN - fgt_begin_algorithm( tbb::internal::FGT_PARALLEL_FOR, (void*)&a ); + fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&a ); + fgt_end_algorithm( (void*)&context ); // END REGION } } #endif /* __TBB_TASK_GROUP_CONTEXT */ //! Run body for range, serves as callback for partitioner void run_body( Range &r ) { - fgt_alg_begin_body( tbb::internal::FGT_PARALLEL_FOR, (void *)const_cast(&(this->my_body)), (void*)this ); + fgt_alg_begin_body( tbb::internal::PARALLEL_FOR_TASK, (void *)const_cast(&(this->my_body)), (void*)this ); my_body( r ); fgt_alg_end_body( (void *)const_cast(&(this->my_body)) ); } diff --git a/src/tbb/include/tbb/parallel_for_each.h b/src/tbb/include/tbb/parallel_for_each.h index 6c2ec9f0..1ce39ece 100644 --- a/src/tbb/include/tbb/parallel_for_each.h +++ b/src/tbb/include/tbb/parallel_for_each.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_for_each_H diff --git a/src/tbb/include/tbb/parallel_invoke.h b/src/tbb/include/tbb/parallel_invoke.h index 0dd7590e..026b445d 100644 --- a/src/tbb/include/tbb/parallel_invoke.h +++ b/src/tbb/include/tbb/parallel_invoke.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_invoke_H #define __TBB_parallel_invoke_H #include "task.h" +#include "tbb_profiling.h" #if __TBB_VARIADIC_PARALLEL_INVOKE #include // std::forward @@ -31,7 +28,9 @@ namespace tbb { #if !__TBB_TASK_GROUP_CONTEXT /** Dummy to avoid cluttering the bulk of the header with enormous amount of ifdefs. **/ - struct task_group_context {}; + struct task_group_context { + task_group_context(tbb::internal::string_index){} + }; #endif /* __TBB_TASK_GROUP_CONTEXT */ //! @cond INTERNAL @@ -223,7 +222,7 @@ namespace internal { // task_group_context is not in the pack, needs to be added template void parallel_invoke_impl(false_type, F0&& f0, F1&& f1, F&&... f) { - tbb::task_group_context context; + tbb::task_group_context context(PARALLEL_INVOKE); // Add context to the arguments, and redirect to the other overload parallel_invoke_impl(true_type(), std::forward(f0), std::forward(f1), std::forward(f)..., context); } @@ -385,31 +384,31 @@ void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, con // two arguments template void parallel_invoke(const F0& f0, const F1& f1) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, context); } // three arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, context); } // four arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, context); } // five arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, f4, context); } // six arguments template void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5) { - task_group_context context; + task_group_context context(internal::PARALLEL_INVOKE); parallel_invoke(f0, f1, f2, f3, f4, f5, context); } // seven arguments @@ -417,7 +416,7 @@ template(f0, f1, f2, f3, f4, f5, f6, context); } // eight arguments @@ -426,7 +425,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, context); } // nine arguments @@ -435,7 +434,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, f8, context); } // ten arguments @@ -444,7 +443,7 @@ template(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, context); } #endif // __TBB_VARIADIC_PARALLEL_INVOKE diff --git a/src/tbb/include/tbb/parallel_reduce.h b/src/tbb/include/tbb/parallel_reduce.h index ab9c9f73..9908de9e 100644 --- a/src/tbb/include/tbb/parallel_reduce.h +++ b/src/tbb/include/tbb/parallel_reduce.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_reduce_H @@ -136,7 +132,7 @@ namespace internal { #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; + task_group_context context(PARALLEL_REDUCE); task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ } @@ -236,10 +232,10 @@ namespace internal { } //! Splitting constructor used to generate children. /** parent_ becomes left child. Newly constructed object is right child. */ - start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c ) : + start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c, typename Partitioner::split_type& split_obj ) : my_body( c.my_right_body ), - my_range( parent_.my_range, split() ), - my_partition( parent_.my_partition, split() ) + my_range( parent_.my_range, split_obj ), + my_partition( parent_.my_partition, split_obj ) { } @@ -251,7 +247,7 @@ namespace internal { #else // Bound context prevents exceptions from body to affect nesting or sibling algorithms, // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context; + task_group_context context(PARALLEL_REDUCE); task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); #endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ } @@ -263,11 +259,11 @@ namespace internal { } #endif /* __TBB_TASK_GROUP_CONTEXT */ - void offer_work( typename Partitioner::split_type& ) { + void offer_work( typename Partitioner::split_type& split_obj) { task* tasks[2]; allocate_sibling(static_cast(this), tasks, sizeof(start_deterministic_reduce), sizeof(finish_type)); new((void*)tasks[0]) finish_type(my_body); - new((void*)tasks[1]) start_deterministic_reduce(*this, *static_cast(tasks[0])); + new((void*)tasks[1]) start_deterministic_reduce(*this, *static_cast(tasks[0]), split_obj); spawn(*tasks[1]); } @@ -393,6 +389,13 @@ void parallel_reduce( const Range& range, Body& body, affinity_partitioner& part } #if __TBB_TASK_GROUP_CONTEXT +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +void parallel_reduce( const Range& range, Body& body, task_group_context& context ) { + internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); +} + //! Parallel iteration with reduction, simple partitioner and user-supplied context. /** @ingroup algorithms **/ template @@ -480,6 +483,17 @@ Value parallel_reduce( const Range& range, const Value& identity, const RealBody } #if __TBB_TASK_GROUP_CONTEXT +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + internal::lambda_reduce_body body(identity, real_body, reduction); + internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); + return body.result(); +} + //! Parallel iteration with reduction, simple partitioner and user-supplied context. /** @ingroup algorithms **/ template diff --git a/src/tbb/include/tbb/parallel_scan.h b/src/tbb/include/tbb/parallel_scan.h index d3668695..7f6478da 100644 --- a/src/tbb/include/tbb/parallel_scan.h +++ b/src/tbb/include/tbb/parallel_scan.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_scan_H diff --git a/src/tbb/include/tbb/parallel_sort.h b/src/tbb/include/tbb/parallel_sort.h index 07d3907b..a543a6aa 100644 --- a/src/tbb/include/tbb/parallel_sort.h +++ b/src/tbb/include/tbb/parallel_sort.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_sort_H @@ -27,6 +23,9 @@ #include #include #include +#if __TBB_TASK_GROUP_CONTEXT + #include "tbb_profiling.h" +#endif namespace tbb { @@ -159,7 +158,7 @@ struct quick_sort_body { template void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { #if __TBB_TASK_GROUP_CONTEXT - task_group_context my_context; + task_group_context my_context(PARALLEL_SORT); const int serial_cutoff = 9; __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); @@ -230,13 +229,6 @@ void parallel_sort(Range& rng, const Compare& comp) { parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); } -//! Sorts the data in const rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - //! Sorts the data in rng with a default comparator \c std::less /** @ingroup algorithms **/ template @@ -244,13 +236,6 @@ void parallel_sort(Range& rng) { parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); } -//! Sorts the data in const rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(const Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - //! Sorts the data in the range \c [begin,end) with a default comparator \c std::less /** @ingroup algorithms **/ template diff --git a/src/tbb/include/tbb/parallel_while.h b/src/tbb/include/tbb/parallel_while.h index 2f37a41a..07177005 100644 --- a/src/tbb/include/tbb/parallel_while.h +++ b/src/tbb/include/tbb/parallel_while.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_parallel_while diff --git a/src/tbb/include/tbb/partitioner.h b/src/tbb/include/tbb/partitioner.h index d7ebdbfe..285ada71 100644 --- a/src/tbb/include/tbb/partitioner.h +++ b/src/tbb/include/tbb/partitioner.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_partitioner_H @@ -50,6 +46,7 @@ #endif // __TBB_DEFINE_MIC #include "task.h" +#include "task_arena.h" #include "aligned_space.h" #include "atomic.h" #include "internal/_template_helpers.h" @@ -257,25 +254,6 @@ struct partition_type_base { } }; -//! Class determines whether template parameter has static boolean constant -//! 'is_splittable_in_proportion' initialized with value of 'true' or not. -/** If template parameter has such field that has been initialized with non-zero -* value then class field will be set to 'true', otherwise - 'false' -*/ -template -class is_splittable_in_proportion { -private: - typedef char yes[1]; - typedef char no[2]; - - template static yes& decide(typename enable_if::type *); - template static no& decide(...); -public: - // equals to 'true' if and only if static const variable 'is_splittable_in_proportion' of template parameter - // initialized with the value of 'true' - static const bool value = (sizeof(decide(0)) == sizeof(yes)); -}; - //! Provides default splitting strategy for partition objects. template struct adaptive_mode : partition_type_base { @@ -294,6 +272,28 @@ struct adaptive_mode : partition_type_base { } }; +//! A helper class to create a proportional_split object for a given type of Range. +/** If the Range has static boolean constant 'is_splittable_in_proportion' set to 'true', + the created object splits a provided value in an implemenation-defined proportion; + otherwise it represents equal-size split. */ +// TODO: check if this helper can be a nested class of proportional_mode. +template +struct proportion_helper { + static proportional_split get_split(size_t) { return proportional_split(1,1); } +}; +template +struct proportion_helper::type> { + static proportional_split get_split(size_t n) { +#if __TBB_NONUNIFORM_TASK_CREATION + size_t right = (n + 2) / 3; +#else + size_t right = n / 2; +#endif + size_t left = n - right; + return proportional_split(left, right); + } +}; + //! Provides proportional splitting strategy for partition objects template struct proportional_mode : adaptive_mode { @@ -324,41 +324,32 @@ struct proportional_mode : adaptive_mode { bool is_divisible() { // part of old should_execute_range() return self().my_divisor > my_partition::factor; } -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif template proportional_split get_split() { - if (is_splittable_in_proportion::value) { - size_t size = self().my_divisor / my_partition::factor; -#if __TBB_NONUNIFORM_TASK_CREATION - size_t right = (size + 2) / 3; -#else - size_t right = size / 2; -#endif - size_t left = size - right; - return proportional_split(left, right); - } else { - return proportional_split(1, 1); - } + // Create a proportion for the number of threads expected to handle "this" subrange + return proportion_helper::get_split( self().my_divisor / my_partition::factor ); } -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back }; +static size_t get_initial_partition_head() { + int current_index = tbb::this_task_arena::current_thread_index(); + if (current_index == tbb::task_arena::not_initialized) + current_index = 0; + return size_t(current_index); +} + //! Provides default linear indexing of partitioner's sequence template struct linear_affinity_mode : proportional_mode { size_t my_head; + size_t my_max_affinity; using proportional_mode::self; - linear_affinity_mode() : proportional_mode(), my_head(0) {} + linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()), + my_max_affinity(self().my_divisor) {} linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) - , my_head(src.my_head + src.my_divisor) {} + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) - , my_head(src.my_head + src.my_divisor) {} + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} void set_affinity( task &t ) { if( self().my_divisor ) t.set_affinity( affinity_id(my_head) + 1 ); @@ -399,7 +390,7 @@ struct dynamic_grainsize_mode : Mode { #endif , my_delay(begin) , my_max_depth(p.my_max_depth) {} - bool check_being_stolen( task &t) { // part of old should_execute_range() + bool check_being_stolen(task &t) { // part of old should_execute_range() if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? if( t.is_stolen_task() && t.parent()->ref_count() >= 2 ) { // runs concurrently with the left task diff --git a/src/tbb/include/tbb/pipeline.h b/src/tbb/include/tbb/pipeline.h index 20a8ec9b..c7138644 100644 --- a/src/tbb/include/tbb/pipeline.h +++ b/src/tbb/include/tbb/pipeline.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_pipeline_H diff --git a/src/tbb/include/tbb/queuing_mutex.h b/src/tbb/include/tbb/queuing_mutex.h index 1dcc87c8..15f94774 100644 --- a/src/tbb/include/tbb/queuing_mutex.h +++ b/src/tbb/include/tbb/queuing_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_queuing_mutex_H @@ -46,6 +42,7 @@ class queuing_mutex : internal::mutex_copy_deprecated_and_disabled { //! Initialize fields to mean "no lock held". void initialize() { mutex = NULL; + going = 0; #if TBB_USE_ASSERT internal::poison_pointer(next); #endif /* TBB_USE_ASSERT */ diff --git a/src/tbb/include/tbb/queuing_rw_mutex.h b/src/tbb/include/tbb/queuing_rw_mutex.h index 702cbe35..6d5d3130 100644 --- a/src/tbb/include/tbb/queuing_rw_mutex.h +++ b/src/tbb/include/tbb/queuing_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_queuing_rw_mutex_H @@ -55,6 +51,8 @@ class queuing_rw_mutex : internal::mutex_copy_deprecated_and_disabled { //! Initialize fields to mean "no lock held". void initialize() { my_mutex = NULL; + my_internal_lock = 0; + my_going = 0; #if TBB_USE_ASSERT my_state = 0xFF; // Set to invalid state internal::poison_pointer(my_next); diff --git a/src/tbb/include/tbb/reader_writer_lock.h b/src/tbb/include/tbb/reader_writer_lock.h index 353beec5..2275abc7 100644 --- a/src/tbb/include/tbb/reader_writer_lock.h +++ b/src/tbb/include/tbb/reader_writer_lock.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_reader_writer_lock_H diff --git a/src/tbb/include/tbb/recursive_mutex.h b/src/tbb/include/tbb/recursive_mutex.h index 5a23c097..576bd999 100644 --- a/src/tbb/include/tbb/recursive_mutex.h +++ b/src/tbb/include/tbb/recursive_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_recursive_mutex_H diff --git a/src/tbb/include/tbb/runtime_loader.h b/src/tbb/include/tbb/runtime_loader.h index df284645..a6d695b3 100644 --- a/src/tbb/include/tbb/runtime_loader.h +++ b/src/tbb/include/tbb/runtime_loader.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_runtime_loader_H diff --git a/src/tbb/include/tbb/scalable_allocator.h b/src/tbb/include/tbb/scalable_allocator.h index 0bebe21e..a077e00c 100644 --- a/src/tbb/include/tbb/scalable_allocator.h +++ b/src/tbb/include/tbb/scalable_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_scalable_allocator_H @@ -95,9 +91,12 @@ typedef enum { TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ /* deprecated, kept for backward compatibility only */ USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value Bytes, clean internal buffers + /* try to limit memory consumption value (Bytes), clean internal buffers if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT + TBBMALLOC_SET_SOFT_HEAP_LIMIT, + /* Lower bound for the size (Bytes), that is interpreted as huge + * and not released during regular cleanup operations. */ + TBBMALLOC_SET_HUGE_SIZE_THRESHOLD } AllocationModeParam; /** Set TBB allocator-specific allocation modes. @@ -193,6 +192,7 @@ void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t ali bool pool_reset(MemoryPool* memPool); bool pool_free(MemoryPool *memPool, void *object); MemoryPool *pool_identify(void *object); +size_t pool_msize(MemoryPool *memPool, void *object); } // namespace rml @@ -208,7 +208,11 @@ MemoryPool *pool_identify(void *object); #endif #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward +#include // std::forward +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include #endif namespace tbb { @@ -315,6 +319,48 @@ inline bool operator==( const scalable_allocator&, const scalable_allocator inline bool operator!=( const scalable_allocator&, const scalable_allocator& ) {return false;} +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +namespace internal { + +//! C++17 memory resource implementation for scalable allocator +//! ISO C++ Section 23.12.2 +class scalable_resource_impl : public std::pmr::memory_resource { +private: + void* do_allocate(size_t bytes, size_t alignment) override { + void* ptr = scalable_aligned_malloc( bytes, alignment ); + if (!ptr) { + throw_exception(std::bad_alloc()); + } + return ptr; + } + + void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override { + scalable_free(ptr); + } + + //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any + //! other instance of this class + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + return this == &other || +#if __TBB_USE_OPTIONAL_RTTI + dynamic_cast(&other) != NULL; +#else + false; +#endif + } +}; + +} // namespace internal + +//! Global scalable allocator memory resource provider +inline std::pmr::memory_resource* scalable_memory_resource() noexcept { + static tbb::internal::scalable_resource_impl scalable_res; + return &scalable_res; +} + +#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ + } // namespace tbb #if _MSC_VER diff --git a/src/tbb/include/tbb/spin_mutex.h b/src/tbb/include/tbb/spin_mutex.h index 99ef15c2..e29cf354 100644 --- a/src/tbb/include/tbb/spin_mutex.h +++ b/src/tbb/include/tbb/spin_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_spin_mutex_H diff --git a/src/tbb/include/tbb/spin_rw_mutex.h b/src/tbb/include/tbb/spin_rw_mutex.h index b20f4ebd..1477e684 100644 --- a/src/tbb/include/tbb/spin_rw_mutex.h +++ b/src/tbb/include/tbb/spin_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_spin_rw_mutex_H @@ -90,9 +86,6 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { class scoped_lock : internal::no_copy { #if __TBB_TSX_AVAILABLE friend class tbb::interface8::internal::x86_rtm_rw_mutex; - // helper methods for x86_rtm_rw_mutex - spin_rw_mutex *internal_get_mutex() const { return mutex; } - void internal_set_mutex(spin_rw_mutex* m) { mutex = m; } #endif public: //! Construct lock that has not acquired a mutex. @@ -121,15 +114,15 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { //! Upgrade reader to become a writer. /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( !is_writer, "not a reader" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); + if (is_writer) return true; // Already a writer is_writer = true; return mutex->internal_upgrade(); } //! Release lock. void release() { - __TBB_ASSERT( mutex, "lock is not acquired" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); spin_rw_mutex *m = mutex; mutex = NULL; #if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT @@ -143,8 +136,8 @@ class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { //! Downgrade writer to become a reader. bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( is_writer, "not a writer" ); + __TBB_ASSERT( mutex, "mutex is not acquired" ); + if (!is_writer) return true; // Already a reader #if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT mutex->internal_downgrade(); #else diff --git a/src/tbb/include/tbb/task.h b/src/tbb/include/tbb/task.h index 0b04c6b9..6eff290f 100644 --- a/src/tbb/include/tbb/task.h +++ b/src/tbb/include/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_H @@ -49,7 +45,9 @@ namespace internal { //< @cond INTERNAL task* self; task& parent; public: - explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) {} + explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) { + suppress_unused_warning( self ); + } task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; void __TBB_EXPORTED_METHOD free( task& ) const; }; @@ -162,6 +160,12 @@ namespace internal { void __TBB_EXPORTED_METHOD free( task& ) const; }; +#if __TBB_PREVIEW_CRITICAL_TASKS + // TODO: move to class methods when critical task API becomes public + void make_critical( task& t ); + bool is_critical( task& t ); +#endif + //! Memory prefix to a task object. /** This class is internal to the library. Do not reference it directly, except within the library itself. @@ -187,6 +191,10 @@ namespace internal { friend class internal::allocate_child_proxy; friend class internal::allocate_continuation_proxy; friend class internal::allocate_additional_child_of_proxy; +#if __TBB_PREVIEW_CRITICAL_TASKS + friend void make_critical( task& ); + friend bool is_critical( task& ); +#endif #if __TBB_TASK_ISOLATION //! The tag used for task isolation. @@ -274,6 +282,10 @@ namespace internal { #if __TBB_TASK_PRIORITY namespace internal { static const int priority_stride_v4 = INT_MAX / 4; +#if __TBB_PREVIEW_CRITICAL_TASKS + // TODO: move into priority_t enum when critical tasks become public feature + static const int priority_critical = priority_stride_v4 * 3 + priority_stride_v4 / 3 * 2; +#endif } enum priority_t { @@ -294,6 +306,7 @@ enum priority_t { class task_scheduler_init; namespace interface7 { class task_arena; } +using interface7::task_arena; //! Used to form groups of tasks /** @ingroup task_scheduling @@ -320,7 +333,7 @@ class task_group_context : internal::no_copy { private: friend class internal::generic_scheduler; friend class task_scheduler_init; - friend class interface7::task_arena; + friend class task_arena; #if TBB_USE_CAPTURED_EXCEPTION typedef tbb_exception exception_container_type; @@ -420,12 +433,16 @@ class task_group_context : internal::no_copy { intptr_t my_priority; #endif /* __TBB_TASK_PRIORITY */ + //! Description of algorithm for scheduler based instrumentation. + internal::string_index my_name; + //! Trailing padding protecting accesses to frequently used members from false sharing /** \sa _leading_padding **/ char _trailing_padding[internal::NFS_MaxLineSize - 2 * sizeof(uintptr_t) - 2 * sizeof(void*) #if __TBB_TASK_PRIORITY - - sizeof(intptr_t) + - sizeof(intptr_t) #endif /* __TBB_TASK_PRIORITY */ + - sizeof(internal::string_index) ]; public: @@ -461,7 +478,17 @@ class task_group_context : internal::no_copy { task_group_context ( kind_type relation_with_parent = bound, uintptr_t t = default_traits ) : my_kind(relation_with_parent) - , my_version_and_traits(2 | t) + , my_version_and_traits(3 | t) + , my_name(internal::CUSTOM_CTX) + { + init(); + } + + // Custom constructor for instrumentation of tbb algorithm + task_group_context ( internal::string_index name ) + : my_kind(bound) + , my_version_and_traits(3 | default_traits) + , my_name(name) { init(); } @@ -783,11 +810,24 @@ class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { #if __TBB_TASK_PRIORITY //! Enqueue task for starvation-resistant execution on the specified priority level. static void enqueue( task& t, priority_t p ) { - __TBB_ASSERT( p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value" ); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); +#else + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); +#endif t.prefix().owner->enqueue( t, (void*)p ); } #endif /* __TBB_TASK_PRIORITY */ + //! Enqueue task in task_arena + //! The implementation is in task_arena.h + inline static void enqueue( task& t, task_arena& arena +#if __TBB_TASK_PRIORITY + , priority_t p = priority_t(0) +#endif + ); + //! The innermost task being executed or destroyed by the current thread at the moment. static task& __TBB_EXPORTED_FUNC self(); @@ -906,8 +946,19 @@ class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { internal::task_prefix& prefix( internal::version_tag* = NULL ) const { return reinterpret_cast(const_cast(this))[-1]; } +#if __TBB_PREVIEW_CRITICAL_TASKS + friend void internal::make_critical( task& ); + friend bool internal::is_critical( task& ); +#endif }; // class task +#if __TBB_PREVIEW_CRITICAL_TASKS +namespace internal { +inline void make_critical( task& t ) { t.prefix().extra_state |= 0x8; } +inline bool is_critical( task& t ) { return bool((t.prefix().extra_state & 0x8) != 0); } +} // namespace internal +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ + //! task that does nothing. Useful for synchronization. /** @ingroup task_scheduling */ class empty_task: public task { @@ -954,7 +1005,7 @@ class task_list: internal::no_copy { //! Destroys the list, but does not destroy the task objects. ~task_list() {} - //! True if list if empty; false otherwise. + //! True if list is empty; false otherwise. bool empty() const {return !first;} //! Push task onto back of list. diff --git a/src/tbb/include/tbb/task_arena.h b/src/tbb/include/tbb/task_arena.h index 94977aeb..f1fef56e 100644 --- a/src/tbb/include/tbb/task_arena.h +++ b/src/tbb/include/tbb/task_arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_arena_H @@ -57,7 +53,7 @@ class delegate_base : no_assign { virtual ~delegate_base() {} }; -// If decltype is availabe, the helper detects the return type of functor of specified type, +// If decltype is available, the helper detects the return type of functor of specified type, // otherwise it defines the void type. template struct return_type_or_void { @@ -96,7 +92,7 @@ class delegated_function : public delegate_base { public: delegated_function(F& f) : my_func(f) {} void consume_result() const {} - + friend class task_arena_base; }; @@ -172,6 +168,11 @@ R isolate_impl(F& f) { */ class task_arena : public internal::task_arena_base { friend class tbb::internal::task_scheduler_observer_v3; + friend void task::enqueue(task&, task_arena& +#if __TBB_TASK_PRIORITY + , priority_t +#endif + ); friend int tbb::this_task_arena::max_concurrency(); bool my_initialized; void mark_initialized() { @@ -313,12 +314,22 @@ class task_arena : public internal::task_arena_base { template #if __TBB_CPP11_RVALUE_REF_PRESENT void enqueue( F&& f, priority_t p ) { +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); +#else __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); +#endif enqueue_impl(std::forward(f), p); } #else void enqueue( const F& f, priority_t p ) { +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high + || p == internal::priority_critical, "Invalid priority level value"); +#else __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); +#endif enqueue_impl(f,p); } #endif @@ -404,6 +415,19 @@ namespace this_task_arena { } } // namespace this_task_arena +//! Enqueue task in task_arena +void task::enqueue( task& t, task_arena& arena +#if __TBB_TASK_PRIORITY + , priority_t p +#endif + ) { +#if !__TBB_TASK_PRIORITY + intptr_t p = 0; +#endif + arena.initialize(); + //! Note: the context of the task may differ from the context instantiated by task_arena + arena.internal_enqueue(t, p); +} } // namespace tbb #endif /* __TBB_task_arena_H */ diff --git a/src/tbb/include/tbb/task_group.h b/src/tbb/include/tbb/task_group.h index b30addc8..cafde9f2 100644 --- a/src/tbb/include/tbb/task_group.h +++ b/src/tbb/include/tbb/task_group.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_group_H @@ -82,6 +78,16 @@ class task_handle_task : public task { }; class task_group_base : internal::no_copy { + class ref_count_guard : internal::no_copy { + task& my_task; + public: + ref_count_guard(task& t) : my_task(t) { + my_task.increment_ref_count(); + } + ~ref_count_guard() { + my_task.decrement_ref_count(); + } + }; protected: empty_task* my_root; task_group_context my_context; @@ -91,8 +97,12 @@ class task_group_base : internal::no_copy { template task_group_status internal_run_and_wait( F& f ) { __TBB_TRY { - if ( !my_context.is_group_execution_cancelled() ) + if ( !my_context.is_group_execution_cancelled() ) { + // We need to increase the reference count of the root task to notify waiters that + // this task group has some work in progress. + ref_count_guard guard(*my_root); f(); + } } __TBB_CATCH( ... ) { my_context.register_pending_exception(); } @@ -114,7 +124,11 @@ class task_group_base : internal::no_copy { ~task_group_base() __TBB_NOEXCEPT(false) { if( my_root->ref_count() > 1 ) { +#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT + bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0; +#else bool stack_unwinding_in_progress = std::uncaught_exception(); +#endif // Always attempt to do proper cleanup to avoid inevitable memory corruption // in case of missing wait (for the sake of better testability & debuggability) if ( !is_canceling() ) diff --git a/src/tbb/include/tbb/task_scheduler_init.h b/src/tbb/include/tbb/task_scheduler_init.h index 928e7a4e..66a24086 100644 --- a/src/tbb/include/tbb/task_scheduler_init.h +++ b/src/tbb/include/tbb/task_scheduler_init.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_scheduler_init_H diff --git a/src/tbb/include/tbb/task_scheduler_observer.h b/src/tbb/include/tbb/task_scheduler_observer.h index 5586ad4f..72694af3 100644 --- a/src/tbb/include/tbb/task_scheduler_observer.h +++ b/src/tbb/include/tbb/task_scheduler_observer.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,17 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_task_scheduler_observer_H #define __TBB_task_scheduler_observer_H #include "atomic.h" -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER #include "task_arena.h" #endif @@ -88,7 +84,7 @@ class task_scheduler_observer_v3 { } // namespace internal -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER namespace interface6 { class task_scheduler_observer : public internal::task_scheduler_observer_v3 { friend class internal::task_scheduler_observer_v3; @@ -149,23 +145,13 @@ class task_scheduler_observer : public internal::task_scheduler_observer_v3 { } internal::task_scheduler_observer_v3::observe(state); } - -#if __TBB_SLEEP_PERMISSION - //! Return commands for may_sleep() - enum { keep_awake = false, allow_sleep = true }; - - //! The callback can be invoked by a worker thread before it goes to sleep. - /** If it returns false ('keep_awake'), the thread will keep spinning and looking for work. - It will not be called for master threads. **/ - virtual bool may_sleep() { return allow_sleep; } -#endif /*__TBB_SLEEP_PERMISSION*/ }; } //namespace interface6 using interface6::task_scheduler_observer; -#else /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ +#else /*__TBB_ARENA_OBSERVER*/ typedef tbb::internal::task_scheduler_observer_v3 task_scheduler_observer; -#endif /*__TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION*/ +#endif /*__TBB_ARENA_OBSERVER*/ } // namespace tbb diff --git a/src/tbb/include/tbb/tbb.h b/src/tbb/include/tbb/tbb.h index 5e385ea2..ba4b1122 100644 --- a/src/tbb/include/tbb/tbb.h +++ b/src/tbb/include/tbb/tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_H @@ -23,7 +19,7 @@ /** This header bulk-includes declarations or definitions of all the functionality - provided by TBB (save for malloc dependent headers). + provided by TBB (save for tbbmalloc and 3rd party dependent headers). If you use only a few TBB constructs, consider including specific headers only. Any header listed below can be included independently of others. @@ -37,6 +33,9 @@ #include "blocked_range.h" #include "blocked_range2d.h" #include "blocked_range3d.h" +#if TBB_PREVIEW_BLOCKED_RANGE_ND +#include "blocked_rangeNd.h" +#endif #include "cache_aligned_allocator.h" #include "combinable.h" #include "concurrent_hash_map.h" @@ -47,13 +46,16 @@ #include "concurrent_queue.h" #include "concurrent_unordered_map.h" #include "concurrent_unordered_set.h" +#if TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#include "concurrent_map.h" +#include "concurrent_set.h" +#endif #include "concurrent_vector.h" #include "critical_section.h" #include "enumerable_thread_specific.h" #include "flow_graph.h" -#if TBB_PREVIEW_GLOBAL_CONTROL #include "global_control.h" -#endif +#include "iterators.h" #include "mutex.h" #include "null_mutex.h" #include "null_rw_mutex.h" diff --git a/src/tbb/include/tbb/tbb_allocator.h b/src/tbb/include/tbb/tbb_allocator.h index 047344e7..20659387 100644 --- a/src/tbb/include/tbb/tbb_allocator.h +++ b/src/tbb/include/tbb/tbb_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_allocator_H @@ -174,7 +170,7 @@ class zero_allocator : public Allocator pointer allocate(const size_type n, const void *hint = 0 ) { pointer ptr = base_allocator_type::allocate( n, hint ); - std::memset( (void*) ptr, 0, n * sizeof(value_type) ); + std::memset( static_cast(ptr), 0, n * sizeof(value_type) ); return ptr; } }; diff --git a/src/tbb/include/tbb/tbb_config.h b/src/tbb/include/tbb/tbb_config.h index 3ac2b3e0..dbfc9fff 100644 --- a/src/tbb/include/tbb/tbb_config.h +++ b/src/tbb/include/tbb/tbb_config.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_config_H @@ -40,23 +36,26 @@ #include #endif -// note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match +// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match // the actual GCC version on the system. #define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -// Since GNU libstdc++ does not have a convenient macro for its version, -// we rely on the version of GCC or the user-specified macro below. -// The format of TBB_USE_GLIBCXX_VERSION should match the __TBB_GCC_VERSION above, -// e.g. it should be set to 40902 for libstdc++ coming with GCC 4.9.2. +// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro. +// Therefore we use different ways to detect its version. #ifdef TBB_USE_GLIBCXX_VERSION +// The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro. +// Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1. #define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION +#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__ +// Reported versions of GCC and libstdc++ do not match; trust the latter +#define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000) #elif __GLIBCPP__ || __GLIBCXX__ +// The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION #define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION -//TODO: analyze __GLIBCXX__ instead of __TBB_GCC_VERSION ? #endif #if __clang__ - /** according to clang documentation, version can be vendor specific **/ + // according to clang documentation, version can be vendor specific #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) #endif @@ -65,6 +64,16 @@ #define __TBB_IOS 1 #endif +#if __APPLE__ + #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \ + && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000 + // ICC does not correctly set the macro if -mmacosx-min-version is not specified + #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000)) + #else + #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ + #endif +#endif + /** Preprocessor symbols to determine HW architecture **/ #if _WIN32||_WIN64 @@ -132,6 +141,16 @@ support added. */ +/** + __TBB_CPP11_PRESENT macro indicates that the compiler supports vast majority of C++11 features. + Depending on the compiler, some features might still be unsupported or work incorrectly. + Use it when enabling C++11 features individually is not practical, and be aware that + some "good enough" compilers might be excluded. **/ +#define __TBB_CPP11_PRESENT (__cplusplus >= 201103L || _MSC_VER >= 1900) + +#define __TBB_CPP17_FALLTHROUGH_PRESENT (__cplusplus >= 201703L) +#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER) + /** C++11 mode detection macros for Intel(R) C++ Compiler (enabled by -std=c++XY option): __INTEL_CXX11_MODE__ for version >=13.0 (not available for ICC 15.0 if -std=c++14 is used), __STDC_HOSTED__ for version >=12.0 (useful only on Windows), @@ -141,16 +160,9 @@ #define __INTEL_CXX11_MODE__ (__GXX_EXPERIMENTAL_CXX0X__ || (_MSC_VER && __STDC_HOSTED__)) #endif -// Intel(R) C++ Compiler offloading API to the Intel(R) Graphics Technology presence macro -// TODO: add support for ICC 15.00 _GFX_enqueue API and then decrease Intel C++ Compiler supported version -// TODO: add linux support and restict it with (__linux__ && __TBB_x86_64 && !__ANDROID__) macro -#if __INTEL_COMPILER >= 1600 && _WIN32 -#define __TBB_GFX_PRESENT 1 -#endif - #if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MODE__) // On Windows, C++11 features supported by Visual Studio 2010 and higher are enabled by default, - // so in absence of /Qstd= use MSVC branch for __TBB_CPP11_* detection. + // so in absence of /Qstd= use MSVC branch for feature detection. // On other platforms, no -std= means C++03. #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (__INTEL_CXX11_MODE__ && __VARIADIC_TEMPLATES) @@ -177,14 +189,7 @@ #endif #define __TBB_STATIC_ASSERT_PRESENT (__INTEL_CXX11_MODE__ || _MSC_VER >= 1600) #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40300 || _LIBCPP_VERSION)) - #if (__clang__ && __INTEL_COMPILER > 1400) - /* Older versions of Intel C++ Compiler do not have __has_include */ - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #else - #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) - #endif + #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) #define __TBB_CONSTEXPR_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) /** ICC seems to disable support of noexcept event in c++11 when compiling in compatibility mode for gcc <4.6 **/ @@ -197,10 +202,12 @@ #define __TBB_OVERRIDE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) #define __TBB_ALIGNAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1500) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 1910) // a future version + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) #elif __clang__ /** TODO: these options need to be rechecked **/ -/** on macOS the only way to get C++11 is to use clang. For library features (e.g. exception_ptr) libc++ is also - * required. So there is no need to check GCC version for clang**/ #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __has_feature(__cxx_variadic_templates__) #define __TBB_CPP11_RVALUE_REF_PRESENT (__has_feature(__cxx_rvalue_references__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40500)) #define __TBB_IMPLICIT_MOVE_PRESENT __has_feature(cxx_implicit_moves) @@ -227,6 +234,10 @@ #define __TBB_OVERRIDE_PRESENT __has_feature(cxx_override_control) #define __TBB_ALIGNAS_PRESENT __has_feature(cxx_alignas) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __has_feature(cxx_alias_templates) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__has_feature(__cpp_deduction_guides)) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__has_feature(__cpp_lib_is_invocable)) #elif __GNUC__ #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __GXX_EXPERIMENTAL_CXX0X__ #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) @@ -251,8 +262,13 @@ #define __TBB_OVERRIDE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) #define __TBB_ALIGNAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40800) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L && __TBB_GCC_VERSION >= 70000) #elif _MSC_VER - // These definitions are also used with Intel C++ Compiler in "default" mode; see a comment above. + // These definitions are also used with Intel C++ Compiler in "default" mode (__INTEL_CXX11_MODE__ == 0); + // see a comment in "__INTEL_COMPILER" section above. #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (_MSC_VER >= 1800) // Contains a workaround for ICC 13 @@ -273,50 +289,72 @@ #define __TBB_OVERRIDE_PRESENT (_MSC_VER >= 1700) #define __TBB_ALIGNAS_PRESENT (_MSC_VER >= 1900) #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (_MSC_VER >= 1800) + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (_MSC_VER >= 1900) + /* Variable templates are supported in VS2015 Update 2 or later */ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1914) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1911) #else - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT 0 - #define __TBB_CPP11_RVALUE_REF_PRESENT 0 - #define __TBB_IMPLICIT_MOVE_PRESENT 0 - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #define __TBB_STATIC_ASSERT_PRESENT 0 - #define __TBB_CPP11_TUPLE_PRESENT 0 - #define __TBB_INITIALIZER_LISTS_PRESENT 0 - #define __TBB_CONSTEXPR_PRESENT 0 - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT 0 - #define __TBB_NOEXCEPT_PRESENT 0 - #define __TBB_CPP11_STD_BEGIN_END_PRESENT 0 - #define __TBB_CPP11_AUTO_PRESENT 0 - #define __TBB_CPP11_DECLTYPE_PRESENT 0 - #define __TBB_CPP11_LAMBDAS_PRESENT 0 - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT 0 - #define __TBB_OVERRIDE_PRESENT 0 - #define __TBB_ALIGNAS_PRESENT 0 - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT 0 + #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_RVALUE_REF_PRESENT __TBB_CPP11_PRESENT + #define __TBB_IMPLICIT_MOVE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_EXCEPTION_PTR_PRESENT __TBB_CPP11_PRESENT + #define __TBB_STATIC_ASSERT_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_TUPLE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_INITIALIZER_LISTS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CONSTEXPR_PRESENT __TBB_CPP11_PRESENT + #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT __TBB_CPP11_PRESENT + #define __TBB_NOEXCEPT_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_STD_BEGIN_END_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_AUTO_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_DECLTYPE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_LAMBDAS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_OVERRIDE_PRESENT __TBB_CPP11_PRESENT + #define __TBB_ALIGNAS_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __TBB_CPP11_PRESENT + #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cplusplus >= 201703L) + #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) #endif // C++11 standard library features +#define __TBB_CPP11_ARRAY_PRESENT (_MSC_VER >= 1700 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300) + #ifndef __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT #endif -#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >=1800) +#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >= 1800) -#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) -#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) +#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) +#define __TBB_TR1_TYPE_PROPERTIES_IN_STD_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300 || _MSC_VER >= 1600) // GCC supported some of type properties since 4.7 -#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) +#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) // In GCC, std::move_if_noexcept appeared later than noexcept -#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) -#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1700 || \ - __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) -#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) +#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) +#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1800 || \ + __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) +#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || __SUNPRO_CC)) // Due to libc++ limitations in C++03 mode, do not pass rvalues to std::make_shared() -#define __TBB_CPP11_SMART_POINTERS_PRESENT ( _MSC_VER >= 1600 || _LIBCPP_VERSION || ((__cplusplus >= 201103L || __GXX_EXPERIMENTAL_CXX0X__) && (__TBB_GLIBCXX_VERSION>=40500 || __TBB_GLIBCXX_VERSION>=40400 && __TBB_USE_OPTIONAL_RTTI)) ) +#define __TBB_CPP11_SMART_POINTERS_PRESENT ( _MSC_VER >= 1600 || _LIBCPP_VERSION \ + || ((__cplusplus >= 201103L || __GXX_EXPERIMENTAL_CXX0X__) \ + && (__TBB_GLIBCXX_VERSION >= 40500 || __TBB_GLIBCXX_VERSION >= 40400 && __TBB_USE_OPTIONAL_RTTI)) ) + +#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) -#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && _GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) +#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) +#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \ + || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200)) +// TODO: wait when memory_resource will be fully supported in clang and define the right macro +// Currently it is in experimental stage since 6 version. +#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (_MSVC_LANG > 201402L || __cplusplus > 201402L) || \ + __GLIBCXX__ && __cpp_lib_memory_resource >= 201603) +#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911) // std::swap is in only since C++11, though MSVC had it at least since VS2005 #if _MSC_VER>=1400 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ #define __TBB_STD_SWAP_HEADER @@ -351,11 +389,21 @@ #define __TBB_GCC_BUILTIN_ATOMICS_PRESENT 1 #endif +#if __TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER && !__clang__ + // After GCC7 there was possible reordering problem in generic atomic load/store operations. + // So always using builtins. + #define TBB_USE_GCC_BUILTINS 1 +#endif + #if __INTEL_COMPILER >= 1200 /** built-in C++11 style atomics available in ICC since 12.0 **/ #define __TBB_ICC_BUILTIN_ATOMICS_PRESENT 1 #endif +#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) + #define __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT 1 +#endif + #define __TBB_TSX_INTRINSICS_PRESENT ((__RTM__ || _MSC_VER>=1700 || __INTEL_COMPILER>=1300) && !__TBB_DEFINE_MIC && !__ANDROID__) /** Macro helpers **/ @@ -461,6 +509,15 @@ There are four cases that are supported: /** Internal TBB features & modes **/ +/** __TBB_CONCURRENT_ORDERED_CONTAINERS indicates that all conditions of use + * concurrent_map and concurrent_set are met. **/ +// TODO: Add cpp11 random generation macro +#ifndef __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + #define __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT ( __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ + && __TBB_IMPLICIT_MOVE_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT && __TBB_CPP11_ARRAY_PRESENT \ + && __TBB_INITIALIZER_LISTS_PRESENT ) +#endif + /** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ #ifndef __TBB_WEAK_SYMBOLS_PRESENT #define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) @@ -502,29 +559,13 @@ There are four cases that are supported: #define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official #ifndef __TBB_ARENA_OBSERVER - #define __TBB_ARENA_OBSERVER ((__TBB_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) + #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER #endif /* __TBB_ARENA_OBSERVER */ -#ifndef __TBB_SLEEP_PERMISSION - #define __TBB_SLEEP_PERMISSION ((__TBB_CPF_BUILD||TBB_PREVIEW_LOCAL_OBSERVER)&& __TBB_SCHEDULER_OBSERVER) -#endif /* __TBB_SLEEP_PERMISSION */ - #ifndef __TBB_TASK_ISOLATION #define __TBB_TASK_ISOLATION 1 #endif /* __TBB_TASK_ISOLATION */ -#if TBB_PREVIEW_FLOW_GRAPH_TRACE || TBB_PREVIEW_ALGORITHM_TRACE -// Users of flow-graph and algorithm trace need to explicitly link against the preview -// library. This prevents the linker from implicitly linking an application with a preview -// version of TBB and unexpectedly bringing in other community preview features, which -// might change the behavior of the application. -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif /* TBB_PREVIEW_FLOW_GRAPH_TRACE */ - -#ifndef __TBB_ITT_STRUCTURE_API -#define __TBB_ITT_STRUCTURE_API ( !__TBB_DEFINE_MIC && (__TBB_CPF_BUILD || TBB_PREVIEW_FLOW_GRAPH_TRACE || TBB_PREVIEW_ALGORITHM_TRACE) ) -#endif - #if TBB_USE_EXCEPTIONS && !__TBB_TASK_GROUP_CONTEXT #error TBB_USE_EXCEPTIONS requires __TBB_TASK_GROUP_CONTEXT to be enabled #endif @@ -576,6 +617,7 @@ There are four cases that are supported: /** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load shared libraries at run time only from application container **/ +// TODO: Separate this single macro into two for Windows 8 Store* (win8ui mode) and UWP/UWD modes. #if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP #define __TBB_WIN8UI_SUPPORT 1 #else @@ -660,7 +702,7 @@ There are four cases that are supported: #define __TBB_ICC_ASM_VOLATILE_BROKEN 1 #endif -#if !__INTEL_COMPILER && (_MSC_VER || __GNUC__==3 && __GNUC_MINOR__<=2) +#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1700 || __GNUC__==3 && __GNUC_MINOR__<=2) /** Bug in GCC 3.2 and MSVC compilers that sometimes return 0 for __alignof(T) when T has not yet been instantiated. **/ #define __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN 1 @@ -739,6 +781,11 @@ There are four cases that are supported: // In some cases decltype of a function adds a reference to a return type. #define __TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN (_MSC_VER == 1600 && !__INTEL_COMPILER) +// Visual Studio 2013 does not delete the copy constructor when a user-defined move constructor is provided +#if _MSC_VER && _MSC_VER <= 1800 + #define __TBB_IMPLICIT_COPY_DELETION_BROKEN 1 +#endif + /** End of __TBB_XXX_BROKEN macro section **/ #if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) @@ -768,7 +815,13 @@ There are four cases that are supported: #define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES || __TBB_PREVIEW_OPENCL_NODE) #define __TBB_PREVIEW_ASYNC_MSG (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_FLOW_GRAPH_CPP11_FEATURES) -#define __TBB_PREVIEW_GFX_FACTORY (__TBB_GFX_PRESENT && TBB_PREVIEW_FLOW_GRAPH_FEATURES && !__TBB_MIC_OFFLOAD \ - && __TBB_FLOW_GRAPH_CPP11_FEATURES && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT \ - && __TBB_CPP11_FUTURE_PRESENT) + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#define __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES TBB_PREVIEW_FLOW_GRAPH_FEATURES +#endif + +#ifndef __TBB_PREVIEW_CRITICAL_TASKS +#define __TBB_PREVIEW_CRITICAL_TASKS (__TBB_CPF_BUILD || __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES) +#endif + #endif /* __TBB_tbb_config_H */ diff --git a/src/tbb/include/tbb/tbb_disable_exceptions.h b/src/tbb/include/tbb/tbb_disable_exceptions.h index a1d28a60..88ef4351 100644 --- a/src/tbb/include/tbb/tbb_disable_exceptions.h +++ b/src/tbb/include/tbb/tbb_disable_exceptions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ //! To disable use of exceptions, include this header before any other header file from the library. diff --git a/src/tbb/include/tbb/tbb_exception.h b/src/tbb/include/tbb/tbb_exception.h index a2f3e1b6..2dcb62b5 100644 --- a/src/tbb/include/tbb/tbb_exception.h +++ b/src/tbb/include/tbb/tbb_exception.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_exception_H diff --git a/src/tbb/include/tbb/tbb_machine.h b/src/tbb/include/tbb/tbb_machine.h index cf8ea04b..095b9d3c 100644 --- a/src/tbb/include/tbb/tbb_machine.h +++ b/src/tbb/include/tbb/tbb_machine.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_machine_H @@ -227,7 +223,7 @@ template<> struct atomic_selector<8> { #include "machine/linux_intel64.h" #endif -#elif __linux__ || __FreeBSD__ || __NetBSD__ +#elif __linux__ || __FreeBSD__ || __NetBSD__ || __OpenBSD__ #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) #include "machine/gcc_generic.h" @@ -241,8 +237,8 @@ template<> struct atomic_selector<8> { #include "machine/linux_ia64.h" #elif __powerpc__ #include "machine/mac_ppc.h" - #elif __ARM_ARCH_7A__ - #include "machine/gcc_armv7.h" + #elif __ARM_ARCH_7A__ || __aarch64__ + #include "machine/gcc_arm.h" #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT #include "machine/gcc_generic.h" #endif @@ -665,7 +661,7 @@ struct machine_load_store_seq_cst { static void store ( volatile T &location, T value ) { #if __TBB_GCC_VERSION >= 40702 #pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif // An atomic initialization leads to reading of uninitialized memory int64_t result = (volatile int64_t&)location; @@ -788,7 +784,7 @@ struct __TBB_machine_type_with_alignment_##PowerOf2 { \ #endif /* Now declare types aligned to useful powers of two */ -// TODO: Is __TBB_DefineTypeWithAlignment(8) needed on 32 bit platforms? +__TBB_DefineTypeWithAlignment(8) // i386 ABI says that uint64_t is aligned on 4 bytes __TBB_DefineTypeWithAlignment(16) __TBB_DefineTypeWithAlignment(32) __TBB_DefineTypeWithAlignment(64) @@ -802,7 +798,7 @@ template struct type_with_alignment; template<> struct type_with_alignment<1> { char member; }; template<> struct type_with_alignment<2> { uint16_t member; }; template<> struct type_with_alignment<4> { uint32_t member; }; -template<> struct type_with_alignment<8> { uint64_t member; }; +template<> struct type_with_alignment<8> { __TBB_machine_type_with_alignment_8 member; }; template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; }; template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; }; template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; }; diff --git a/src/tbb/include/tbb/tbb_profiling.h b/src/tbb/include/tbb/tbb_profiling.h index ffaf98b9..21665800 100644 --- a/src/tbb/include/tbb/tbb_profiling.h +++ b/src/tbb/include/tbb/tbb_profiling.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_profiling_H @@ -24,10 +20,6 @@ namespace tbb { namespace internal { - // - // This is not under __TBB_ITT_STRUCTURE_API because these values are used directly in flow_graph.h. - // - // include list of index names #define TBB_STRING_RESOURCE(index_name,str) index_name, enum string_index { @@ -138,8 +130,7 @@ namespace tbb { void __TBB_EXPORTED_FUNC itt_store_pointer_with_release_v3(void *dst, void *src); void* __TBB_EXPORTED_FUNC itt_load_pointer_with_acquire_v3(const void *src); void* __TBB_EXPORTED_FUNC itt_load_pointer_v3( const void* src ); -#if __TBB_ITT_STRUCTURE_API - enum itt_domain_enum { ITT_DOMAIN_FLOW=0 }; + enum itt_domain_enum { ITT_DOMAIN_FLOW=0, ITT_DOMAIN_MAIN=1, ITT_DOMAIN_ALGO=2, ITT_NUM_DOMAINS }; void __TBB_EXPORTED_FUNC itt_make_task_group_v7( itt_domain_enum domain, void *group, unsigned long long group_extra, void *parent, unsigned long long parent_extra, string_index name_index ); @@ -154,7 +145,6 @@ namespace tbb { void __TBB_EXPORTED_FUNC itt_region_begin_v9( itt_domain_enum domain, void *region, unsigned long long region_extra, void *parent, unsigned long long parent_extra, string_index name_index ); void __TBB_EXPORTED_FUNC itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long region_extra ); -#endif // __TBB_ITT_STRUCTURE_API // two template arguments are to workaround /Wp64 warning with tbb::atomic specialized for unsigned type template @@ -238,12 +228,6 @@ namespace tbb { call_itt_notify_v5((int)t, ptr); } -#else - inline void call_itt_notify(notify_type /*t*/, void * /*ptr*/) {} - -#endif // TBB_USE_THREADING_TOOLS - -#if __TBB_ITT_STRUCTURE_API inline void itt_make_task_group( itt_domain_enum domain, void *group, unsigned long long group_extra, void *parent, unsigned long long parent_extra, string_index name_index ) { itt_make_task_group_v7( domain, group, group_extra, parent, parent_extra, name_index ); @@ -276,9 +260,81 @@ namespace tbb { inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { itt_region_end_v9( domain, region, region_extra ); } -#endif // __TBB_ITT_STRUCTURE_API +#else + inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {} + + inline void itt_make_task_group( itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_metadata_str_add( itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, + string_index /*key*/, const char* /*value*/ ) {} + + inline void itt_relation_add( itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/, + itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) {} + + inline void itt_task_begin( itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_task_end( itt_domain_enum /*domain*/ ) {} + + inline void itt_region_begin( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} + + inline void itt_region_end( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) {} +#endif // TBB_USE_THREADING_TOOLS } // namespace internal } // namespace tbb +#if TBB_PREVIEW_FLOW_GRAPH_TRACE +#include + +namespace tbb { +namespace profiling { +namespace interface10 { + +#if TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) +class event { +/** This class supports user event traces through itt. + Common use-case is tagging data flow graph tasks (data-id) + and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/ +// TODO: Replace implementation by itt user event api. + + const std::string my_name; + + static void emit_trace(const std::string &input) { + itt_metadata_str_add( tbb::internal::ITT_DOMAIN_FLOW, NULL, tbb::internal::FLOW_NULL, tbb::internal::USER_EVENT, ( "FGA::DATAID::" + input ).c_str() ); + } + +public: + event(const std::string &input) + : my_name( input ) + { } + + void emit() { + emit_trace(my_name); + } + + static void emit(const std::string &description) { + emit_trace(description); + } + +}; +#else // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) +// Using empty struct if user event tracing is disabled: +struct event { + event(const std::string &) { } + + void emit() { } + + static void emit(const std::string &) { } +}; +#endif // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) + +} // interfaceX +using interface10::event; +} // namespace profiling +} // namespace tbb +#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE + #endif /* __TBB_profiling_H */ diff --git a/src/tbb/include/tbb/tbb_stddef.h b/src/tbb/include/tbb/tbb_stddef.h index b4ee7810..bb3233db 100644 --- a/src/tbb/include/tbb/tbb_stddef.h +++ b/src/tbb/include/tbb/tbb_stddef.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,21 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_stddef_H #define __TBB_tbb_stddef_H // Marketing-driven product version -#define TBB_VERSION_MAJOR 2018 +#define TBB_VERSION_MAJOR 2019 #define TBB_VERSION_MINOR 0 // Engineering-focused interface version -#define TBB_INTERFACE_VERSION 10001 +#define TBB_INTERFACE_VERSION 11008 #define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 // The oldest major interface version still supported @@ -155,8 +151,8 @@ namespace tbb { #if TBB_USE_ASSERT - //! Assert that x is true. - /** If x is false, print assertion failure message. + //! Assert that predicate is true. + /** If predicate is false, print assertion failure message. If the comment argument is not NULL, it is printed as part of the failure message. The comment argument has no other effect. */ #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) @@ -246,6 +242,14 @@ const size_t NFS_MaxLineSize = 128; #define __TBB_override // formal comment only #endif +#if __TBB_CPP17_FALLTHROUGH_PRESENT +#define __TBB_fallthrough [[fallthrough]] +#elif __TBB_FALLTHROUGH_PRESENT +#define __TBB_fallthrough __attribute__ ((fallthrough)) +#else +#define __TBB_fallthrough +#endif + template struct padded_base : T { char pad[S - R]; @@ -326,7 +330,7 @@ class no_assign { //! Base class for types that should not be copied or assigned. class no_copy: no_assign { //! Deny copy construction - no_copy( const no_copy& ); + //no_copy( const no_copy& ); public: //! Allow default construction no_copy() {} @@ -423,8 +427,8 @@ class proportional_split: internal::no_assign { // Following is a set of classes and functions typically used in compile-time "metaprogramming". // TODO: move all that to a separate header -#if __TBB_ALLOCATOR_TRAITS_PRESENT || __TBB_CPP11_SMART_POINTERS_PRESENT -#include // for allocator_traits, unique_ptr +#if __TBB_CPP11_SMART_POINTERS_PRESENT +#include // for unique_ptr #endif #if __TBB_CPP11_RVALUE_REF_PRESENT || __TBB_CPP11_DECLTYPE_PRESENT || _LIBCPP_VERSION @@ -464,15 +468,6 @@ struct bool_constant { typedef bool_constant true_type; typedef bool_constant false_type; -#if __TBB_ALLOCATOR_TRAITS_PRESENT -using std::allocator_traits; -#else -template -struct allocator_traits{ - typedef tbb::internal::false_type propagate_on_container_move_assignment; -}; -#endif - //! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. template struct select_size_t_constant { @@ -533,7 +528,7 @@ struct STATIC_ASSERTION_FAILED; //intentionally left undefined to cause co //! @endcond }} // namespace tbb::internal -#if __TBB_STATIC_ASSERT_PRESENT +#if __TBB_STATIC_ASSERT_PRESENT #define __TBB_STATIC_ASSERT(condition,msg) static_assert(condition,msg) #else //please note condition is intentionally inverted to get a bit more understandable error msg diff --git a/src/tbb/include/tbb/tbb_thread.h b/src/tbb/include/tbb/tbb_thread.h index c2846b9e..52b96e4d 100644 --- a/src/tbb/include/tbb/tbb_thread.h +++ b/src/tbb/include/tbb/tbb_thread.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_thread_H @@ -246,7 +242,7 @@ namespace internal { friend tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); friend inline size_t tbb_hasher( const tbb_thread_v3::id& id ) { - __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementaion assumes that thread_id_type fits into machine word"); + __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementation assumes that thread_id_type fits into machine word"); return tbb::tbb_hasher(id.my_id); } diff --git a/src/tbb/include/tbb/tbbmalloc_proxy.h b/src/tbb/include/tbb/tbbmalloc_proxy.h index 76cbd6d7..d7b7235a 100644 --- a/src/tbb/include/tbb/tbbmalloc_proxy.h +++ b/src/tbb/include/tbb/tbbmalloc_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* @@ -63,4 +59,7 @@ struct __TBB_malloc_proxy_caller { #endif // _MSC_VER +/* Public Windows API */ +extern "C" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr); + #endif //__TBB_tbbmalloc_proxy_H diff --git a/src/tbb/include/tbb/tick_count.h b/src/tbb/include/tbb/tick_count.h index a7f4e0f3..a3c744ce 100644 --- a/src/tbb/include/tbb/tick_count.h +++ b/src/tbb/include/tbb/tick_count.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tick_count_H diff --git a/src/tbb/index.html b/src/tbb/index.html new file mode 100644 index 00000000..d8d3eeaf --- /dev/null +++ b/src/tbb/index.html @@ -0,0 +1,48 @@ + + +

Overview

+Top level directory for Intel® Threading Building Blocks (Intel® TBB). +

Common directories

+ +

Intel TBB source package

+

+To build Intel TBB, use the top-level Makefile; see also the build directions. +To port Intel TBB to a new platform, operating system or architecture, see the porting directions. +

+

Files

+
+
Makefile +
Top-level Makefile for Intel TBB. See also the build directions. +
+

Directories

+
+
src +
Source code for the library. +
build, jni +
Internal Makefile infrastructure for Intel TBB. Do not use directly; see the build directions. +
+

Intel TBB binary package

+

Directories

+
+
bin +
Start-up scripts for sourcing library for Linux* OS and macOS*. For Windows* OS: start-up scripts and dynamic-link libraries. +
lib +
Platform-specific binary files for the library. +
+
+

+Copyright © 2005-2019 Intel Corporation. All Rights Reserved. +

+Intel is a registered trademark or trademark of Intel Corporation +or its subsidiaries in the United States and other countries. +

+* Other names and brands may be claimed as the property of others. + + diff --git a/src/tbb/jni/Android.mk b/src/tbb/jni/Android.mk new file mode 100644 index 00000000..522bc84c --- /dev/null +++ b/src/tbb/jni/Android.mk @@ -0,0 +1,62 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export tbb_root?=$(NDK_PROJECT_PATH) + +ifeq (armeabi-v7a,$(APP_ABI)) + export SYSROOT:=$(NDK_ROOT)/platforms/$(APP_PLATFORM)/arch-arm +else ifeq (arm64-v8a,$(APP_ABI)) + export SYSROOT:=$(NDK_ROOT)/platforms/$(APP_PLATFORM)/arch-arm64 +else + export SYSROOT:=$(NDK_ROOT)/platforms/$(APP_PLATFORM)/arch-$(APP_ABI) +endif + +ifeq (windows,$(tbb_os)) + export CPATH_SEPARATOR :=; +else + export CPATH_SEPARATOR :=: +endif + +export ANDROID_NDK_ROOT:=$(NDK_ROOT) +export ndk_version:=$(lastword $(subst -, ,$(ANDROID_NDK_ROOT))) +ndk_version:= $(firstword $(subst /, ,$(ndk_version))) +ndk_version:= $(firstword $(subst \, ,$(ndk_version))) + +ifeq (clang,$(compiler)) + ifeq (,$(findstring $(ndk_version),ifeq (,$(findstring $(ndk_version),$(foreach v, 7 8 9 10 11 12,r$(v) r$(v)b r$(v)c r$(v)d r$(v)e))))) + TBB_RTL :=llvm-libc++ + else + TBB_RTL :=llvm-libc++/libcxx + endif + TBB_RTL_LIB :=llvm-libc++ + TBB_RTL_FILE :=libc++_shared.so +else + TBB_RTL :=gnu-libstdc++/$(NDK_TOOLCHAIN_VERSION) + TBB_RTL_LIB :=$(TBB_RTL) + TBB_RTL_FILE :=libgnustl_shared.so +endif + +export CPATH := $(SYSROOT)/usr/include$(CPATH_SEPARATOR)$(NDK_ROOT)/sources/cxx-stl/$(TBB_RTL)/include$(CPATH_SEPARATOR)$(NDK_ROOT)/sources/cxx-stl/$(TBB_RTL)/libs/$(APP_ABI)/include$(CPATH_SEPARATOR)$(NDK_ROOT)/sources/android/support/include + +LIB_STL_ANDROID_DIR := $(NDK_ROOT)/sources/cxx-stl/$(TBB_RTL_LIB)/libs/$(APP_ABI) +#LIB_STL_ANDROID is required to be set up for copying Android specific library to a device next to test +export LIB_STL_ANDROID := $(LIB_STL_ANDROID_DIR)/$(TBB_RTL_FILE) +export CPLUS_LIB_PATH := $(SYSROOT)/usr/lib -L$(LIB_STL_ANDROID_DIR) +export target_os_version:=$(APP_PLATFORM) +export tbb_tool_prefix:=$(TOOLCHAIN_PREFIX) +export TARGET_CXX +export TARGET_CC +export TARGET_CFLAGS + +include $(NDK_PROJECT_PATH)/src/Makefile diff --git a/src/tbb/jni/Application.mk b/src/tbb/jni/Application.mk new file mode 100644 index 00000000..80ace2f6 --- /dev/null +++ b/src/tbb/jni/Application.mk @@ -0,0 +1,67 @@ +# Copyright (c) 2005-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifndef tbb_os + + # Windows sets environment variable OS; for other systems, ask uname + ifeq ($(OS),) + OS:=$(shell uname) + ifeq ($(OS),) + $(error "Cannot detect operating system") + endif + export tbb_os=$(OS) + endif + + ifeq ($(OS), Windows_NT) + export tbb_os=windows + endif + ifeq ($(OS), Linux) + export tbb_os=linux + endif + ifeq ($(OS), Darwin) + export tbb_os=macos + endif + +endif + +export compiler?=clang +export arch?=ia32 +export target?=android + +ifeq (ia32,$(arch)) + APP_ABI:=x86 + export TRIPLE:=i686-linux-android +else ifeq (intel64,$(arch)) + APP_ABI:=x86_64 + export TRIPLE:=x86_64-linux-android +else ifeq (arm,$(arch)) + APP_ABI:=armeabi-v7a + export TRIPLE:=arm-linux-androideabi +else ifeq (arm64,$(arch)) + APP_ABI:=arm64-v8a + export TRIPLE:=aarch64-linux-android +else + APP_ABI:=$(arch) +endif + +api_version?=21 +export API_LEVEL:=$(api_version) +APP_PLATFORM:=android-$(api_version) + +ifeq (clang,$(compiler)) + NDK_TOOLCHAIN_VERSION:=clang + APP_STL:=c++_shared +else + NDK_TOOLCHAIN_VERSION:=4.9 +endif diff --git a/src/tbb/python/Makefile b/src/tbb/python/Makefile new file mode 100644 index 00000000..ca7beb9c --- /dev/null +++ b/src/tbb/python/Makefile @@ -0,0 +1,45 @@ +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tbb_root?=.. +BUILDING_PHASE:=0 +include $(tbb_root)/build/common.inc +.PHONY: all release test install test-install + +export TBBROOT=$(abspath $(tbb_root)) +SRC=$(tbb_root)/python/*.py $(tbb_root)/python/tbb/* +PY_SETUP=python $(tbb_root)/python/setup.py + +all: install test + +clean: + $(PY_SETUP) clean -b$(work_dir)_release + +release: CC=$(compiler) +release: $(SRC) rml + $(PY_SETUP) build -b$(work_dir)_release -f check + +install: CC=$(compiler) +install: $(SRC) rml + $(PY_SETUP) build -b$(work_dir)_release build_ext -f -I$(tbb_root)/include -L$(work_dir)_release install -f + +test: + python -m tbb test + +rml: +ifeq (linux,$(tbb_os)) + $(MAKE) -C "$(work_dir)_release" -rf $(tbb_root)/python/rml/Makefile cfg=release rml +rml_%: + $(MAKE) -C "$(work_dir)_release" -rf $(tbb_root)/python/rml/Makefile cfg=release $(subst rml_,,$@) +endif diff --git a/src/tbb/python/TBB.py b/src/tbb/python/TBB.py new file mode 100644 index 00000000..0a20dd0b --- /dev/null +++ b/src/tbb/python/TBB.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tbb import * +from tbb import __all__, __doc__ + +if __name__ == "__main__": + from tbb import _main + import sys + sys.exit(_main()) diff --git a/src/tbb/python/index.html b/src/tbb/python/index.html new file mode 100644 index 00000000..b8446190 --- /dev/null +++ b/src/tbb/python/index.html @@ -0,0 +1,84 @@ + + +

Python* API for Intel® Threading Building Blocks (Intel® TBB). +

+ +

Overview

+It is a preview Python* module which unlocks opportunities for additional performance in multi-threaded and multiprocess Python programs by enabling threading composability +between two or more thread-enabled libraries like Numpy, Scipy, Sklearn, Dask, Joblib, and etc. +

+The biggest improvement can be achieved when a task pool like the ThreadPool or Pool from the Python standard library or libraries like Dask or Joblib (used either in multi-threading or multi-processing mode) +execute tasks calling compute-intensive functions of Numpy/Scipy/Sklearn/PyDAAL which in turn are parallelized using Intel® Math Kernel Library or/and Intel® TBB. +

+The module implements Pool class with the standard interface using Intel® TBB which can be used to replace Python's ThreadPool. +Thanks to the monkey-patching technique implemented in class Monkey, no source code change is needed in order to enable threading composability in Python programs. +

+For more information and examples, please refer to online blog. + +

Directories

+
+
rml +
The folder contains sources for building the plugin with cross-process dynamic thread scheduler implementation. +
tbb +
The folder contains Python module sources. +
+ +

Files

+
+
setup.py +
Standard Python setup script. +
Makefile +
Internal Makefile for building, installing, and testing. See below. +
TBB.py +
Alternative entry point for Python module. +
+ +

Build and install (source package only)

 +For accessing targets defined in python/Makefile, please use +src/Makefile +instead and build runtime libraries before working with Python. +
+
make -C ../src python_all +
Install and test as described below. +
make -C ../src python_install +
Install module into Python environment. +
make -C ../src python_test +
Test installed Intel® TBB module for Python. +
make -C ../src python_release +
Recompile Python module. Result is located in Intel® TBB build directory. +
make -C ../src python_clean +
Remove any intermediate files produced by the commands above. Does not remove installed module. +
+ +

Command-line interface

+
+
python -m tbb -h +
Print documentation on command-line interface
+
pydoc tbb +
Read built-in documentation for Python interfaces.
+
python-tbb your_script.py +
python -m tbb your_script.py +
Run your_script.py in context of `with tbb.Monkey():` when Intel® TBB is enabled. By default only multi-threading will be covered.
+
python -m tbb --ipc your_script.py +
Run your_script.py in context of `with tbb.Monkey():` when Intel® TBB enabled in both multi-threading and multi-processing modes.
+
+ +

System Requirements

+The Python module was not tested on older versions of Python thus we require at least Python versions 2.7 and 3.5 or higher.
+SWIG must be of version 3.0.6 or higher
+OS versions: +Microsoft* Windows* Server 2012, +Microsoft* Windows* 10, +Ubuntu* 14.04 LTS, +Red Hat* Enterprise Linux* 7. +
+Up to parent directory +

+Copyright © 2016-2019 Intel Corporation. All Rights Reserved. +

+Intel is a registered trademark or trademark of Intel Corporation +or its subsidiaries in the United States and other countries. +

+* Other names and brands may be claimed as the property of others. + + diff --git a/src/tbb/python/rml/Makefile b/src/tbb/python/rml/Makefile new file mode 100644 index 00000000..2bfaf65c --- /dev/null +++ b/src/tbb/python/rml/Makefile @@ -0,0 +1,151 @@ +# Copyright (c) 2017-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +.NOTPARALLEL: + +tbb_root ?= ../.. +BUILDING_PHASE=0 +TEST_RESOURCE = $(RML.RES) +include $(tbb_root)/build/common.inc +DEBUG_SUFFIX=$(findstring _debug,_$(cfg)) + +ifneq (linux,$(target)) +$(error "IPC RML is supported on Linux only") +endif + +.PHONY: default rml test clean + +# default target +default: rml test + +RML_ROOT ?= $(tbb_root)/src/rml +RML_SERVER_ROOT = $(RML_ROOT)/server +# TODO: new API needs to be added for this server, exposing everything +RML.DEF = + +VPATH = $(tbb_root)/src/tbb $(tbb_root)/src/tbb/$(ASSEMBLY_SOURCE) +VPATH += $(tbb_root)/python/rml $(RML_ROOT)/test $(tbb_root)/src/test +VPATH += $(tbb_root)/src/rml/client + +include $(tbb_root)/build/common_rules.inc + +#-------------------------------------------------------------------------- +# Define rules for making the RML server shared library and client objects. +#-------------------------------------------------------------------------- + +# Object files that make up RML server +RML_SERVER.OBJ = ipc_server.$(OBJ) + +# Object files that RML clients need +RML_TBB_CLIENT.OBJ ?= ipc_utils.$(OBJ) +RML.OBJ = $(RML_SERVER.OBJ) $(RML_TBB_CLIENT.OBJ) +ifeq (windows,$(tbb_os)) +RML_ASM.OBJ = $(if $(findstring intel64,$(arch)),$(TBB_ASM.OBJ)) +endif +ifeq (linux,$(tbb_os)) +RML_ASM.OBJ = $(if $(findstring ia64,$(arch)),$(TBB_ASM.OBJ)) +endif + +RML_TBB_DEP= cache_aligned_allocator_rml.$(OBJ) dynamic_link_rml.$(OBJ) tbb_misc_rml.$(OBJ) tbb_misc_ex_rml.$(OBJ) +TBB_DEP_NON_RML_TEST?= cache_aligned_allocator_rml.$(OBJ) dynamic_link_rml.$(OBJ) $(RML_ASM.OBJ) tbb_misc_rml.$(OBJ) tbb_misc_ex_rml.$(OBJ) +ifeq ($(cfg),debug) +RML_TBB_DEP+= spin_mutex_rml.$(OBJ) +TBB_DEP_RML_TEST?= $(RML_ASM.OBJ) tbb_misc_rml.$(OBJ) +else +TBB_DEP_RML_TEST?= $(RML_ASM.OBJ) +endif +LIBS += $(LIBDL) +TBB_DEP_RML_TEST = rml_tbb.$(OBJ) dynamic_link_rml.$(OBJ) + +INCLUDES += $(INCLUDE_KEY)$(RML_ROOT)/include $(INCLUDE_KEY). +T_INCLUDES = $(INCLUDES) $(INCLUDE_KEY)$(tbb_root)/src/test $(INCLUDE_KEY)$(RML_SERVER_ROOT) + +# Suppress superfluous warnings for RML compilation +R_CPLUS_FLAGS = $(subst DO_ITT_NOTIFY,DO_ITT_NOTIFY=0,$(CPLUS_FLAGS)) $(WARNING_SUPPRESS) \ + $(DEFINE_KEY)TBB_USE_THREADING_TOOLS=0 $(DEFINE_KEY)__TBB_RML_STATIC=1 $(DEFINE_KEY)__TBB_NO_IMPLICIT_LINKAGE=1 + +%.$(OBJ): %.cpp + $(CPLUS) $(COMPILE_ONLY) $(R_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(INCLUDES) $< + +tbb_misc_rml.$(OBJ) $(RML_SERVER.OBJ): version_string.ver + +RML_TEST.OBJ = test_job_automaton.$(OBJ) test_thread_monitor.$(OBJ) test_rml_tbb.$(OBJ) + +$(RML_TBB_DEP): %_rml.$(OBJ): %.cpp + $(CPLUS) $(COMPILE_ONLY) $(OUTPUTOBJ_KEY)$@ $(R_CPLUS_FLAGS) $(PIC_KEY) $(DSE_KEY) $(INCLUDES) $< + +$(RML_TEST.OBJ): %.$(OBJ): %.cpp + $(CPLUS) $(COMPILE_ONLY) $(R_CPLUS_FLAGS) $(PIC_KEY) $(T_INCLUDES) $< + +ifneq (,$(RML.DEF)) +rml.def: $(RML.DEF) + $(CPLUS) $(PREPROC_ONLY) $< $(CPLUS_FLAGS) $(INCLUDES) > $@ + +LIB_LINK_FLAGS += $(EXPORT_KEY)rml.def +$(RML.DLL): rml.def +endif + +$(RML.DLL): CPLUS_FLAGS += $(SDL_FLAGS) +$(RML.DLL): BUILDING_LIBRARY = $(RML.DLL) +$(RML.DLL): $(RML_TBB_DEP) $(RML.OBJ) $(RML.RES) $(RML_NO_VERSION.DLL) $(RML_ASM.OBJ) + $(LIB_LINK_CMD) $(LIB_OUTPUT_KEY)$(RML.DLL) $(RML.OBJ) $(RML_TBB_DEP) $(RML_ASM.OBJ) $(RML.RES) $(LIB_LINK_LIBS) $(LIB_LINK_FLAGS) + +ifneq (,$(RML_NO_VERSION.DLL)) +$(RML_NO_VERSION.DLL): + echo "INPUT ($(RML.DLL))" > $(RML_NO_VERSION.DLL) +endif + +rml: rml_dll +rml_dll: $(RML.DLL) + +#------------------------------------------------------ +# End of rules for making the RML server shared library +#------------------------------------------------------ + +#------------------------------------------------------ +# Define rules for making the RML unit tests +#------------------------------------------------------ + +add_debug=$(basename $(1))_debug$(suffix $(1)) +cross_suffix=$(if $(crosstest),$(if $(DEBUG_SUFFIX),$(subst _debug,,$(1)),$(call add_debug,$(1))),$(1)) + +RML_TESTS = test_job_automaton.$(TEST_EXT) test_thread_monitor.$(TEST_EXT) +RML_CUSTOM_TESTS = test_rml_tbb.$(TEST_EXT) + +test_rml_tbb.$(TEST_EXT): test_rml_tbb.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(TBB_DEP_RML_TEST) + $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) test_rml_tbb.$(OBJ) $(RML_TBB_CLIENT.OBJ) $(TBB_DEP_RML_TEST) $(LIBS) $(LINK_FLAGS) + +$(RML_TESTS): %.$(TEST_EXT): %.$(OBJ) $(TBB_DEP_NON_RML_TEST) + $(CPLUS) $(OUTPUT_KEY)$@ $(CPLUS_FLAGS) $< $(TBB_DEP_NON_RML_TEST) $(LIBS) $(LINK_FLAGS) + +export IPC_ENABLE=1 +### run_cmd is usually empty +test: $(call cross_suffix,$(RML.DLL)) $(TEST_PREREQUISITE) $(RML_TESTS) $(RML_CUSTOM_TESTS) + $(run_cmd) ./test_job_automaton.$(TEST_EXT) $(args) + $(run_cmd) ./test_thread_monitor.$(TEST_EXT) $(args) +#TODO: $(run_cmd) ./test_rml_tbb.$(TEST_EXT) $(args) +#TODO: IPC_ENABLE=1 LD_PRELOAD=$(abspath libirml.so.1) $(MAKE) -rf $(tbb_root)/src/Makefile cfg=release tbb_test_release + +#------------------------------------------------------ +# End of rules for making the TBBMalloc unit tests +#------------------------------------------------------ + +# Include automatically generated dependencies +-include *.d + +clean: + -rm -rf *.o *.so* *.d *.def version_string.ver + -rm -rf $(work_dir)_release/libirml* + -rm -rf $(work_dir)_debug/libirml* diff --git a/src/tbb/python/rml/ipc_server.cpp b/src/tbb/python/rml/ipc_server.cpp new file mode 100644 index 00000000..50824894 --- /dev/null +++ b/src/tbb/python/rml/ipc_server.cpp @@ -0,0 +1,1115 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "rml_tbb.h" +#include "../server/thread_monitor.h" +#include "tbb/atomic.h" +#include "tbb/cache_aligned_allocator.h" +#include "tbb/scheduler_common.h" +#include "tbb/governor.h" +#include "tbb/tbb_misc.h" + +#include "ipc_utils.h" + +#include + +namespace rml { +namespace internal { + +static const char* IPC_ENABLE_VAR_NAME = "IPC_ENABLE"; + +typedef versioned_object::version_type version_type; + +extern "C" factory::status_type __RML_open_factory(factory& f, version_type& /*server_version*/, version_type /*client_version*/) { + if( !tbb::internal::rml::get_enable_flag( IPC_ENABLE_VAR_NAME ) ) { + return factory::st_incompatible; + } + + // Hack to keep this library from being closed + static tbb::atomic one_time_flag; + if( one_time_flag.compare_and_swap(true,false)==false ) { + __TBB_ASSERT( (size_t)f.library_handle!=factory::c_dont_unload, NULL ); +#if _WIN32||_WIN64 + f.library_handle = reinterpret_cast(factory::c_dont_unload); +#else + f.library_handle = reinterpret_cast(factory::c_dont_unload); +#endif + } + // End of hack + + return factory::st_success; +} + +extern "C" void __RML_close_factory(factory& /*f*/) { +} + +class ipc_thread_monitor : public thread_monitor { +public: + ipc_thread_monitor() : thread_monitor() {} + +#if USE_WINTHREAD +#elif USE_PTHREAD + static handle_type launch(thread_routine_type thread_routine, void* arg, size_t stack_size); +#endif +}; + +#if USE_WINTHREAD +#elif USE_PTHREAD +inline ipc_thread_monitor::handle_type ipc_thread_monitor::launch(void* (*thread_routine)(void*), void* arg, size_t stack_size) { + pthread_attr_t s; + if( pthread_attr_init( &s ) ) return 0; + if( stack_size>0 ) { + if( pthread_attr_setstacksize( &s, stack_size ) ) return 0; + } + pthread_t handle; + if( pthread_create( &handle, &s, thread_routine, arg ) ) return 0; + if( pthread_attr_destroy( &s ) ) return 0; + return handle; +} +#endif + +}} //rml::internal + +using rml::internal::ipc_thread_monitor; + +namespace tbb { +namespace internal { +namespace rml { + +typedef ipc_thread_monitor::handle_type thread_handle; + +class ipc_server; + +static const char* IPC_MAX_THREADS_VAR_NAME = "MAX_THREADS"; +static const char* IPC_ACTIVE_SEM_PREFIX = "/__IPC_active"; +static const char* IPC_STOP_SEM_PREFIX = "/__IPC_stop"; +static const char* IPC_ACTIVE_SEM_VAR_NAME = "IPC_ACTIVE_SEMAPHORE"; +static const char* IPC_STOP_SEM_VAR_NAME = "IPC_STOP_SEMAPHORE"; +static const mode_t IPC_SEM_MODE = 0660; + +static tbb::atomic my_global_thread_count; + +char* get_active_sem_name() { + char* value = getenv( IPC_ACTIVE_SEM_VAR_NAME ); + if( value!=NULL && strlen( value )>0 ) { + char* sem_name = new char[strlen( value ) + 1]; + __TBB_ASSERT( sem_name!=NULL, NULL ); + strcpy( sem_name, value ); + return sem_name; + } else { + return get_shared_name( IPC_ACTIVE_SEM_PREFIX ); + } +} + +char* get_stop_sem_name() { + char* value = getenv( IPC_STOP_SEM_VAR_NAME ); + if( value!=NULL && strlen( value )>0 ) { + char* sem_name = new char[strlen( value ) + 1]; + __TBB_ASSERT( sem_name!=NULL, NULL ); + strcpy( sem_name, value ); + return sem_name; + } else { + return get_shared_name( IPC_STOP_SEM_PREFIX ); + } +} + +static void release_thread_sem(sem_t* my_sem) { + int old; + do { + old = my_global_thread_count; + if( old<=0 ) return; + } while( my_global_thread_count.compare_and_swap(old-1, old)!=old ); + if( old>0 ) { + sem_post( my_sem ); + } +} + +extern "C" void set_active_sem_name() { + char* templ = new char[strlen( IPC_ACTIVE_SEM_PREFIX ) + strlen( "_XXXXXX" ) + 1]; + __TBB_ASSERT( templ!=NULL, NULL ); + strcpy( templ, IPC_ACTIVE_SEM_PREFIX ); + strcpy( templ + strlen( IPC_ACTIVE_SEM_PREFIX ), "_XXXXXX" ); + char* sem_name = mktemp( templ ); + if( sem_name!=NULL ) { + int status = setenv( IPC_ACTIVE_SEM_VAR_NAME, sem_name, 1 ); + __TBB_ASSERT_EX( status==0, NULL ); + } + delete[] templ; +} + +extern "C" void set_stop_sem_name() { + char* templ = new char[strlen( IPC_STOP_SEM_PREFIX ) + strlen( "_XXXXXX" ) + 1]; + __TBB_ASSERT( templ!=NULL, NULL ); + strcpy( templ, IPC_STOP_SEM_PREFIX ); + strcpy( templ + strlen( IPC_STOP_SEM_PREFIX ), "_XXXXXX" ); + char* sem_name = mktemp( templ ); + if( sem_name!=NULL ) { + int status = setenv( IPC_STOP_SEM_VAR_NAME, sem_name, 1 ); + __TBB_ASSERT_EX( status==0, NULL ); + } + delete[] templ; +} + +extern "C" void release_resources() { + if( my_global_thread_count!=0 ) { + char* active_sem_name = get_active_sem_name(); + sem_t* my_active_sem = sem_open( active_sem_name, O_CREAT ); + __TBB_ASSERT( my_active_sem, "Unable to open active threads semaphore" ); + delete[] active_sem_name; + + do { + release_thread_sem( my_active_sem ); + } while( my_global_thread_count!=0 ); + } +} + +extern "C" void release_semaphores() { + int status = 0; + char* sem_name = NULL; + + sem_name = get_active_sem_name(); + if( sem_name==NULL ) { + runtime_warning("Can not get RML semaphore name"); + return; + } + status = sem_unlink( sem_name ); + if( status!=0 ) { + if( errno==ENOENT ) { + /* There is no semaphore with the given name, nothing to do */ + } else { + runtime_warning("Can not release RML semaphore"); + return; + } + } + delete[] sem_name; + + sem_name = get_stop_sem_name(); + if( sem_name==NULL ) { + runtime_warning( "Can not get RML semaphore name" ); + return; + } + status = sem_unlink( sem_name ); + if( status!=0 ) { + if( errno==ENOENT ) { + /* There is no semaphore with the given name, nothing to do */ + } else { + runtime_warning("Can not release RML semaphore"); + return; + } + } + delete[] sem_name; +} + +class ipc_worker: no_copy { +protected: + //! State in finite-state machine that controls the worker. + /** State diagram: + /----------stop---\ + | ^ | + V | | + init --> starting --> normal | + | | | | + | V | | + \------> quit <-------/<----/ + */ + enum state_t { + //! *this is initialized + st_init, + //! *this has associated thread that is starting up. + st_starting, + //! Associated thread is doing normal life sequence. + st_normal, + //! Associated thread is stopped but can be started again. + st_stop, + //! Associated thread has ended normal life sequence and promises to never touch *this again. + st_quit + }; + atomic my_state; + + //! Associated server + ipc_server& my_server; + + //! Associated client + tbb_client& my_client; + + //! index used for avoiding the 64K aliasing problem + const size_t my_index; + + //! Monitor for sleeping when there is no work to do. + /** The invariant that holds for sleeping workers is: + "my_slack<=0 && my_state==st_normal && I am on server's list of asleep threads" */ + ipc_thread_monitor my_thread_monitor; + + //! Handle of the OS thread associated with this worker + thread_handle my_handle; + + //! Link for list of workers that are sleeping or have no associated thread. + ipc_worker* my_next; + + friend class ipc_server; + + //! Actions executed by the associated thread + void run(); + + //! Wake up associated thread (or launch a thread if there is none) + bool wake_or_launch(); + + //! Called by a thread (usually not the associated thread) to commence termination. + void start_shutdown(bool join); + + //! Called by a thread (usually not the associated thread) to commence stopping. + void start_stopping(bool join); + + static __RML_DECL_THREAD_ROUTINE thread_routine(void* arg); + + static void release_handle(thread_handle my_handle, bool join); + +protected: + ipc_worker(ipc_server& server, tbb_client& client, const size_t i) : + my_server(server), + my_client(client), + my_index(i) + { + my_state = st_init; + } +}; + +static const size_t cache_line_size = tbb::internal::NFS_MaxLineSize; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress overzealous compiler warnings about uninstantiable class + #pragma warning(push) + #pragma warning(disable:4510 4610) +#endif +class padded_ipc_worker: public ipc_worker { + char pad[cache_line_size - sizeof(ipc_worker)%cache_line_size]; +public: + padded_ipc_worker(ipc_server& server, tbb_client& client, const size_t i) + : ipc_worker( server,client,i ) { suppress_unused_warning(pad); } +}; +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + +class ipc_waker : public padded_ipc_worker { +private: + static __RML_DECL_THREAD_ROUTINE thread_routine(void* arg); + void run(); + bool wake_or_launch(); + + friend class ipc_server; + +public: + ipc_waker(ipc_server& server, tbb_client& client, const size_t i) + : padded_ipc_worker( server, client, i ) {} +}; + +class ipc_stopper : public padded_ipc_worker { +private: + static __RML_DECL_THREAD_ROUTINE thread_routine(void* arg); + void run(); + bool wake_or_launch(); + + friend class ipc_server; + +public: + ipc_stopper(ipc_server& server, tbb_client& client, const size_t i) + : padded_ipc_worker( server, client, i ) {} +}; + +class ipc_server: public tbb_server, no_copy { +private: + tbb_client& my_client; + //! Maximum number of threads to be created. + /** Threads are created lazily, so maximum might not actually be reached. */ + tbb_client::size_type my_n_thread; + + //! Stack size for each thread. */ + const size_t my_stack_size; + + //! Number of jobs that could use their associated thread minus number of active threads. + /** If negative, indicates oversubscription. + If positive, indicates that more threads should run. + Can be lowered asynchronously, but must be raised only while holding my_asleep_list_mutex, + because raising it impacts the invariant for sleeping threads. */ + atomic my_slack; + + //! Counter used to determine when to delete this. + atomic my_ref_count; + + padded_ipc_worker* my_thread_array; + + //! List of workers that are asleep or committed to sleeping until notified by another thread. + tbb::atomic my_asleep_list_root; + + //! Protects my_asleep_list_root + typedef scheduler_mutex_type asleep_list_mutex_type; + asleep_list_mutex_type my_asleep_list_mutex; + + //! Should server wait workers while terminate + const bool my_join_workers; + + //! Service thread for waking of workers + ipc_waker* my_waker; + + //! Service thread to stop threads + ipc_stopper* my_stopper; + + //! Semaphore to account active threads + sem_t* my_active_sem; + + //! Semaphore to account stop threads + sem_t* my_stop_sem; + +#if TBB_USE_ASSERT + atomic my_net_slack_requests; +#endif /* TBB_USE_ASSERT */ + + //! Wake up to two sleeping workers, if there are any sleeping. + /** The call is used to propagate a chain reaction where each thread wakes up two threads, + which in turn each wake up two threads, etc. */ + void propagate_chain_reaction() { + // First test of a double-check idiom. Second test is inside wake_some(0). + if( my_slack>0 ) { + int active_threads = 0; + if( try_get_active_thread() ) { + ++active_threads; + if( try_get_active_thread() ) { + ++active_threads; + } + wake_some( 0, active_threads ); + } + } + } + + //! Try to add t to list of sleeping workers + bool try_insert_in_asleep_list(ipc_worker& t); + + //! Try to add t to list of sleeping workers even if there is some work to do + bool try_insert_in_asleep_list_forced(ipc_worker& t); + + //! Equivalent of adding additional_slack to my_slack and waking up to 2 threads if my_slack permits. + void wake_some(int additional_slack, int active_threads); + + //! Equivalent of adding additional_slack to my_slack and waking up to 1 thread if my_slack permits. + void wake_one_forced(int additional_slack); + + //! Stop one thread from asleep list + bool stop_one(); + + //! Wait for active thread + bool wait_active_thread(); + + //! Try to get active thread + bool try_get_active_thread(); + + //! Release active thread + void release_active_thread(); + + //! Wait for thread to stop + bool wait_stop_thread(); + + //! Add thread to stop list + void add_stop_thread(); + + void remove_server_ref() { + if( --my_ref_count==0 ) { + my_client.acknowledge_close_connection(); + this->~ipc_server(); + tbb::cache_aligned_allocator().deallocate( this, 1 ); + } + } + + friend class ipc_worker; + friend class ipc_waker; + friend class ipc_stopper; +public: + ipc_server(tbb_client& client); + virtual ~ipc_server(); + + version_type version() const __TBB_override { + return 0; + } + + void request_close_connection(bool /*exiting*/) __TBB_override { + my_waker->start_shutdown(false); + my_stopper->start_shutdown(false); + for( size_t i=0; i=2 && !__MINGW64__ +// ensure that stack is properly aligned +__attribute__((force_align_arg_pointer)) +#endif +__RML_DECL_THREAD_ROUTINE ipc_worker::thread_routine(void* arg) { + ipc_worker* self = static_cast(arg); + AVOID_64K_ALIASING( self->my_index ); + self->run(); + return 0; +} +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + +void ipc_worker::release_handle(thread_handle handle, bool join) { + if( join ) + ipc_thread_monitor::join( handle ); + else + ipc_thread_monitor::detach_thread( handle ); +} + +void ipc_worker::start_shutdown(bool join) { + state_t s; + + do { + s = my_state; + __TBB_ASSERT( s!=st_quit, NULL ); + } while( my_state.compare_and_swap( st_quit, s )!=s ); + if( s==st_normal || s==st_starting ) { + // May have invalidated invariant for sleeping, so wake up the thread. + // Note that the notify() here occurs without maintaining invariants for my_slack. + // It does not matter, because my_state==st_quit overrides checking of my_slack. + my_thread_monitor.notify(); + // Do not need release handle in st_init state, + // because in this case the thread wasn't started yet. + // For st_starting release is done at launch site. + if( s==st_normal ) + release_handle( my_handle, join ); + } +} + +void ipc_worker::start_stopping(bool join) { + state_t s; + + do { + s = my_state; + } while( my_state.compare_and_swap( st_stop, s )!=s ); + if( s==st_normal || s==st_starting ) { + // May have invalidated invariant for sleeping, so wake up the thread. + // Note that the notify() here occurs without maintaining invariants for my_slack. + // It does not matter, because my_state==st_quit overrides checking of my_slack. + my_thread_monitor.notify(); + // Do not need release handle in st_init state, + // because in this case the thread wasn't started yet. + // For st_starting release is done at launch site. + if( s==st_normal ) + release_handle( my_handle, join ); + } +} + +void ipc_worker::run() { + my_server.propagate_chain_reaction(); + + // Transiting to st_normal here would require setting my_handle, + // which would create race with the launching thread and + // complications in handle management on Windows. + + ::rml::job& j = *my_client.create_one_job(); + state_t state = my_state; + while( state!=st_quit && state!=st_stop ) { + if( my_server.my_slack>=0 ) { + my_client.process(j); + } else { + ipc_thread_monitor::cookie c; + // Prepare to wait + my_thread_monitor.prepare_wait(c); + // Check/set the invariant for sleeping + state = my_state; + if( state!=st_quit && state!=st_stop && my_server.try_insert_in_asleep_list(*this) ) { + if( my_server.my_n_thread > 1 ) my_server.release_active_thread(); + my_thread_monitor.commit_wait(c); + my_server.propagate_chain_reaction(); + } else { + // Invariant broken + my_thread_monitor.cancel_wait(); + } + } + state = my_state; + } + my_client.cleanup(j); + + my_server.remove_server_ref(); +} + +inline bool ipc_worker::wake_or_launch() { + if( ( my_state==st_init && my_state.compare_and_swap( st_starting, st_init )==st_init ) || + ( my_state==st_stop && my_state.compare_and_swap( st_starting, st_stop )==st_stop ) ) { + // after this point, remove_server_ref() must be done by created thread +#if USE_WINTHREAD + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size, &this->my_index ); +#elif USE_PTHREAD + { + affinity_helper fpa; + fpa.protect_affinity_mask( /*restore_process_mask=*/true ); + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size ); + if( my_handle == 0 ) { + // Unable to create new thread for process + // However, this is expected situation for the use cases of this coordination server + state_t s = my_state.compare_and_swap( st_init, st_starting ); + if (st_starting != s) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + return false; + } else { + my_server.my_ref_count++; + } + // Implicit destruction of fpa resets original affinity mask. + } +#endif /* USE_PTHREAD */ + state_t s = my_state.compare_and_swap( st_normal, st_starting ); + if( st_starting!=s ) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + } + else { + my_thread_monitor.notify(); + } + + return true; +} + +//------------------------------------------------------------------------ +// Methods of ipc_waker +//------------------------------------------------------------------------ +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress overzealous compiler warnings about an initialized variable 'sink_for_alloca' not referenced + #pragma warning(push) + #pragma warning(disable:4189) +#endif +#if __MINGW32__ && __GNUC__==4 &&__GNUC_MINOR__>=2 && !__MINGW64__ +// ensure that stack is properly aligned +__attribute__((force_align_arg_pointer)) +#endif +__RML_DECL_THREAD_ROUTINE ipc_waker::thread_routine(void* arg) { + ipc_waker* self = static_cast(arg); + AVOID_64K_ALIASING( self->my_index ); + self->run(); + return 0; +} +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + +void ipc_waker::run() { + // Transiting to st_normal here would require setting my_handle, + // which would create race with the launching thread and + // complications in handle management on Windows. + + while( my_state!=st_quit ) { + bool have_to_sleep = false; + if( my_server.my_slack>0 ) { + if( my_server.wait_active_thread() ) { + if( my_server.my_slack>0 ) { + my_server.wake_some( 0, 1 ); + } else { + my_server.release_active_thread(); + have_to_sleep = true; + } + } + } else { + have_to_sleep = true; + } + if( have_to_sleep ) { + ipc_thread_monitor::cookie c; + // Prepare to wait + my_thread_monitor.prepare_wait(c); + // Check/set the invariant for sleeping + if( my_state!=st_quit && my_server.my_slack<0 ) { + my_thread_monitor.commit_wait(c); + } else { + // Invariant broken + my_thread_monitor.cancel_wait(); + } + } + } + + my_server.remove_server_ref(); +} + +inline bool ipc_waker::wake_or_launch() { + if( my_state==st_init && my_state.compare_and_swap( st_starting, st_init )==st_init ) { + // after this point, remove_server_ref() must be done by created thread +#if USE_WINTHREAD + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size, &this->my_index ); +#elif USE_PTHREAD + { + affinity_helper fpa; + fpa.protect_affinity_mask( /*restore_process_mask=*/true ); + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size ); + if( my_handle == 0 ) { + runtime_warning( "Unable to create new thread for process %d", getpid() ); + state_t s = my_state.compare_and_swap( st_init, st_starting ); + if (st_starting != s) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + return false; + } else { + my_server.my_ref_count++; + } + // Implicit destruction of fpa resets original affinity mask. + } +#endif /* USE_PTHREAD */ + state_t s = my_state.compare_and_swap( st_normal, st_starting ); + if( st_starting!=s ) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + } + else { + my_thread_monitor.notify(); + } + + return true; +} + +//------------------------------------------------------------------------ +// Methods of ipc_stopper +//------------------------------------------------------------------------ +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress overzealous compiler warnings about an initialized variable 'sink_for_alloca' not referenced + #pragma warning(push) + #pragma warning(disable:4189) +#endif +#if __MINGW32__ && __GNUC__==4 &&__GNUC_MINOR__>=2 && !__MINGW64__ +// ensure that stack is properly aligned +__attribute__((force_align_arg_pointer)) +#endif +__RML_DECL_THREAD_ROUTINE ipc_stopper::thread_routine(void* arg) { + ipc_stopper* self = static_cast(arg); + AVOID_64K_ALIASING( self->my_index ); + self->run(); + return 0; +} +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + +void ipc_stopper::run() { + // Transiting to st_normal here would require setting my_handle, + // which would create race with the launching thread and + // complications in handle management on Windows. + + while( my_state!=st_quit ) { + if( my_server.wait_stop_thread() ) { + if( my_state!=st_quit ) { + if( !my_server.stop_one() ) { + my_server.add_stop_thread(); + prolonged_pause(); + } + } + } + } + + my_server.remove_server_ref(); +} + +inline bool ipc_stopper::wake_or_launch() { + if( my_state==st_init && my_state.compare_and_swap( st_starting, st_init )==st_init ) { + // after this point, remove_server_ref() must be done by created thread +#if USE_WINTHREAD + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size, &this->my_index ); +#elif USE_PTHREAD + { + affinity_helper fpa; + fpa.protect_affinity_mask( /*restore_process_mask=*/true ); + my_handle = ipc_thread_monitor::launch( thread_routine, this, my_server.my_stack_size ); + if( my_handle == 0 ) { + runtime_warning( "Unable to create new thread for process %d", getpid() ); + state_t s = my_state.compare_and_swap( st_init, st_starting ); + if (st_starting != s) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + return false; + } else { + my_server.my_ref_count++; + } + // Implicit destruction of fpa resets original affinity mask. + } +#endif /* USE_PTHREAD */ + state_t s = my_state.compare_and_swap( st_normal, st_starting ); + if( st_starting!=s ) { + // Do shutdown during startup. my_handle can't be released + // by start_shutdown, because my_handle value might be not set yet + // at time of transition from st_starting to st_quit. + __TBB_ASSERT( s==st_quit, NULL ); + release_handle( my_handle, my_server.my_join_workers ); + } + } + else { + my_thread_monitor.notify(); + } + + return true; +} + +//------------------------------------------------------------------------ +// Methods of ipc_server +//------------------------------------------------------------------------ +ipc_server::ipc_server(tbb_client& client) : + my_client( client ), + my_stack_size( client.min_stack_size() ), + my_thread_array(NULL), + my_join_workers(false), + my_waker(NULL), + my_stopper(NULL) +{ + my_ref_count = 1; + my_slack = 0; +#if TBB_USE_ASSERT + my_net_slack_requests = 0; +#endif /* TBB_USE_ASSERT */ + my_n_thread = get_num_threads(IPC_MAX_THREADS_VAR_NAME); + if( my_n_thread==0 ) { + my_n_thread = AvailableHwConcurrency(); + __TBB_ASSERT( my_n_thread>0, NULL ); + } + + my_asleep_list_root = NULL; + my_thread_array = tbb::cache_aligned_allocator().allocate( my_n_thread ); + memset( my_thread_array, 0, sizeof(padded_ipc_worker)*my_n_thread ); + for( size_t i=0; imy_next = my_asleep_list_root; + my_asleep_list_root = t; + } + + my_waker = tbb::cache_aligned_allocator().allocate(1); + memset( my_waker, 0, sizeof(ipc_waker) ); + new( my_waker ) ipc_waker( *this, client, my_n_thread ); + + my_stopper = tbb::cache_aligned_allocator().allocate(1); + memset( my_stopper, 0, sizeof(ipc_stopper) ); + new( my_stopper ) ipc_stopper( *this, client, my_n_thread + 1 ); + + char* active_sem_name = get_active_sem_name(); + my_active_sem = sem_open( active_sem_name, O_CREAT, IPC_SEM_MODE, my_n_thread - 1 ); + __TBB_ASSERT( my_active_sem, "Unable to open active threads semaphore" ); + delete[] active_sem_name; + + char* stop_sem_name = get_stop_sem_name(); + my_stop_sem = sem_open( stop_sem_name, O_CREAT, IPC_SEM_MODE, 0 ); + __TBB_ASSERT( my_stop_sem, "Unable to open stop threads semaphore" ); + delete[] stop_sem_name; +} + +ipc_server::~ipc_server() { + __TBB_ASSERT( my_net_slack_requests==0, NULL ); + + for( size_t i=my_n_thread; i--; ) + my_thread_array[i].~padded_ipc_worker(); + tbb::cache_aligned_allocator().deallocate( my_thread_array, my_n_thread ); + tbb::internal::poison_pointer( my_thread_array ); + + my_waker->~ipc_waker(); + tbb::cache_aligned_allocator().deallocate( my_waker, 1 ); + tbb::internal::poison_pointer( my_waker ); + + my_stopper->~ipc_stopper(); + tbb::cache_aligned_allocator().deallocate( my_stopper, 1 ); + tbb::internal::poison_pointer( my_stopper ); + + sem_close( my_active_sem ); + sem_close( my_stop_sem ); +} + +inline bool ipc_server::try_insert_in_asleep_list(ipc_worker& t) { + asleep_list_mutex_type::scoped_lock lock; + if( !lock.try_acquire( my_asleep_list_mutex ) ) + return false; + // Contribute to slack under lock so that if another takes that unit of slack, + // it sees us sleeping on the list and wakes us up. + int k = ++my_slack; + if( k<=0 ) { + t.my_next = my_asleep_list_root; + my_asleep_list_root = &t; + return true; + } else { + --my_slack; + return false; + } +} + +inline bool ipc_server::try_insert_in_asleep_list_forced(ipc_worker& t) { + asleep_list_mutex_type::scoped_lock lock; + if( !lock.try_acquire( my_asleep_list_mutex ) ) + return false; + // Contribute to slack under lock so that if another takes that unit of slack, + // it sees us sleeping on the list and wakes us up. + ++my_slack; + t.my_next = my_asleep_list_root; + my_asleep_list_root = &t; + return true; +} + +inline bool ipc_server::wait_active_thread() { + if( sem_wait( my_active_sem ) == 0 ) { + ++my_global_thread_count; + return true; + } + return false; +} + +inline bool ipc_server::try_get_active_thread() { + if( sem_trywait( my_active_sem ) == 0 ) { + ++my_global_thread_count; + return true; + } + return false; +} + +inline void ipc_server::release_active_thread() { + release_thread_sem( my_active_sem ); +} + +inline bool ipc_server::wait_stop_thread() { + struct timespec ts; + if( clock_gettime( CLOCK_REALTIME, &ts )==0 ) { + ts.tv_sec++; + if( sem_timedwait( my_stop_sem, &ts )==0 ) { + return true; + } + } + return false; +} + +inline void ipc_server::add_stop_thread() { + sem_post( my_stop_sem ); +} + +void ipc_server::wake_some( int additional_slack, int active_threads ) { + __TBB_ASSERT( additional_slack>=0, NULL ); + ipc_worker* wakee[2]; + ipc_worker **w = wakee; + { + asleep_list_mutex_type::scoped_lock lock(my_asleep_list_mutex); + while( active_threads>0 && my_asleep_list_root && w0 ) { + if( additional_slack+my_slack<=0 ) // additional demand does not exceed surplus supply + break; + --additional_slack; + } else { + // Chain reaction; Try to claim unit of slack + int old; + do { + old = my_slack; + if( old<=0 ) goto done; + } while( my_slack.compare_and_swap( old-1, old )!=old ); + } + // Pop sleeping worker to combine with claimed unit of slack + my_asleep_list_root = (*w++ = my_asleep_list_root)->my_next; + --active_threads; + } + if( additional_slack ) { + // Contribute our unused slack to my_slack. + my_slack += additional_slack; + } + } +done: + while( w>wakee ) { + if( !(*--w)->wake_or_launch() ) { + add_stop_thread(); + do { + } while( !try_insert_in_asleep_list_forced(**w) ); + release_active_thread(); + } + } + while( active_threads ) { + release_active_thread(); + --active_threads; + } +} + +void ipc_server::wake_one_forced( int additional_slack ) { + __TBB_ASSERT( additional_slack>=0, NULL ); + ipc_worker* wakee[1]; + ipc_worker **w = wakee; + { + asleep_list_mutex_type::scoped_lock lock(my_asleep_list_mutex); + while( my_asleep_list_root && w0 ) { + if( additional_slack+my_slack<=0 ) // additional demand does not exceed surplus supply + break; + --additional_slack; + } else { + // Chain reaction; Try to claim unit of slack + int old; + do { + old = my_slack; + if( old<=0 ) goto done; + } while( my_slack.compare_and_swap( old-1, old )!=old ); + } + // Pop sleeping worker to combine with claimed unit of slack + my_asleep_list_root = (*w++ = my_asleep_list_root)->my_next; + } + if( additional_slack ) { + // Contribute our unused slack to my_slack. + my_slack += additional_slack; + } + } +done: + while( w>wakee ) { + if( !(*--w)->wake_or_launch() ) { + add_stop_thread(); + do { + } while( !try_insert_in_asleep_list_forced(**w) ); + } + } +} + +bool ipc_server::stop_one() { + ipc_worker* current = NULL; + ipc_worker* next = NULL; + { + asleep_list_mutex_type::scoped_lock lock(my_asleep_list_mutex); + if( my_asleep_list_root ) { + current = my_asleep_list_root; + if( current->my_state==ipc_worker::st_normal ) { + next = current->my_next; + while( next!= NULL && next->my_state==ipc_worker::st_normal ) { + current = next; + next = current->my_next; + } + current->start_stopping( my_join_workers ); + return true; + } + } + } + return false; +} + +void ipc_server::adjust_job_count_estimate( int delta ) { +#if TBB_USE_ASSERT + my_net_slack_requests+=delta; +#endif /* TBB_USE_ASSERT */ + if( my_n_thread > 1 ) { + if( delta<0 ) { + my_slack+=delta; + } else if( delta>0 ) { + int active_threads = 0; + if( try_get_active_thread() ) { + ++active_threads; + if( try_get_active_thread() ) { + ++active_threads; + } + } + wake_some( delta, active_threads ); + + if( !my_waker->wake_or_launch() ) { + add_stop_thread(); + } + if( !my_stopper->wake_or_launch() ) { + add_stop_thread(); + } + } + } else { // Corner case when RML shouldn't provide any worker thread but client has to have at least one + if( delta<0 ) { + my_slack += delta; + } else { + wake_one_forced( delta ); + } + } +} + +//------------------------------------------------------------------------ +// RML factory methods +//------------------------------------------------------------------------ + +#if USE_PTHREAD + +static tbb_client* my_global_client = NULL; +static tbb_server* my_global_server = NULL; + +void rml_atexit() { + release_resources(); +} + +void rml_atfork_child() { + if( my_global_server!=NULL && my_global_client!=NULL ) { + ipc_server* server = static_cast( my_global_server ); + server->~ipc_server(); + memset( server, 0, sizeof(ipc_server) ); + new( server ) ipc_server( *my_global_client ); + pthread_atfork( NULL, NULL, rml_atfork_child ); + atexit( rml_atexit ); + } +} + +#endif /* USE_PTHREAD */ + +extern "C" tbb_factory::status_type __TBB_make_rml_server(tbb_factory& /*f*/, tbb_server*& server, tbb_client& client) { + server = new( tbb::cache_aligned_allocator().allocate(1) ) ipc_server(client); +#if USE_PTHREAD + my_global_client = &client; + my_global_server = server; + pthread_atfork( NULL, NULL, rml_atfork_child ); + atexit( rml_atexit ); +#endif /* USE_PTHREAD */ + if( getenv( "RML_DEBUG" ) ) { + runtime_warning("IPC server is started"); + } + return tbb_factory::st_success; +} + +extern "C" void __TBB_call_with_my_server_info(::rml::server_info_callback_t /*cb*/, void* /*arg*/) { +} + +} // namespace rml +} // namespace internal + +} // namespace tbb diff --git a/src/tbb/python/rml/ipc_utils.cpp b/src/tbb/python/rml/ipc_utils.cpp new file mode 100644 index 00000000..c46f6820 --- /dev/null +++ b/src/tbb/python/rml/ipc_utils.cpp @@ -0,0 +1,140 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ipc_utils.h" + +#include +#include +#include +#include +#include + +namespace tbb { +namespace internal { +namespace rml { + +#define MAX_STR_LEN 255 +#define STARTTIME_ITEM_ID 21 + +static char* get_stat_item(char* line, int item_id) { + int id = 0, i = 0; + + while( id!=item_id ) { + while( line[i]!='(' && line[i]!=' ' && line[i]!='\0' ) { + ++i; + } + if( line[i]==' ' ) { + ++id; + ++i; + } else if( line[i]=='(' ) { + while( line[i]!=')' && line[i]!='\0' ) { + ++i; + } + if( line[i]==')' ) { + ++i; + } else { + return NULL; + } + } else { + return NULL; + } + } + + return line + i; +} + +unsigned long long get_start_time(int pid) { + const char* stat_file_path_template = "/proc/%d/stat"; + char stat_file_path[MAX_STR_LEN + 1]; + sprintf( stat_file_path, stat_file_path_template, pid ); + + FILE* stat_file = fopen( stat_file_path, "rt" ); + if( stat_file==NULL ) { + return 0; + } + + char stat_line[MAX_STR_LEN + 1]; + char* line = fgets( stat_line, MAX_STR_LEN, stat_file ); + if( line==NULL ) { + return 0; + } + + char* starttime_str = get_stat_item( stat_line, STARTTIME_ITEM_ID ); + if( starttime_str==NULL ) { + return 0; + } + + unsigned long long starttime = strtoull( starttime_str, NULL, 10 ); + if( starttime==ULLONG_MAX ) { + return 0; + } + + return starttime; +} + +char* get_shared_name(const char* prefix, int pid, unsigned long long time) { + const char* name_template = "%s_%d_%llu"; + const int digits_in_int = 10; + const int digits_in_long = 20; + + int len = strlen( name_template ) + strlen( prefix ) + digits_in_int + digits_in_long + 1; + char* name = new char[len]; + sprintf( name, name_template, prefix, pid, time ); + + return name; +} + +char* get_shared_name(const char* prefix) { + int pid = getpgrp(); + unsigned long long time = get_start_time( pid ); + return get_shared_name( prefix, pid, time ); +} + +int get_num_threads(const char* env_var) { + if( env_var==NULL ) { + return 0; + } + + char* value = getenv( env_var ); + if( value==NULL ) { + return 0; + } + + int num_threads = (int)strtol( value, NULL, 10 ); + return num_threads; +} + +bool get_enable_flag(const char* env_var) { + if( env_var==NULL ) { + return false; + } + + char* value = getenv( env_var ); + if( value==NULL ) { + return false; + } + + if( strcmp( value, "0" ) == 0 || + strcmp( value, "false" ) == 0 || + strcmp( value, "False" ) == 0 || + strcmp( value, "FALSE" ) == 0 ) { + return false; + } + + return true; +} + +}}} //tbb::internal::rml diff --git a/src/tbb/python/rml/ipc_utils.h b/src/tbb/python/rml/ipc_utils.h new file mode 100644 index 00000000..92de5afd --- /dev/null +++ b/src/tbb/python/rml/ipc_utils.h @@ -0,0 +1,30 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __IPC_UTILS_H +#define __IPC_UTILS_H + +namespace tbb { +namespace internal { +namespace rml { + +char* get_shared_name(const char* prefix); +int get_num_threads(const char* env_var); +bool get_enable_flag(const char* env_var); + +}}} //tbb::internal::rml + +#endif diff --git a/src/tbb/python/setup.py b/src/tbb/python/setup.py new file mode 100644 index 00000000..7792f654 --- /dev/null +++ b/src/tbb/python/setup.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# System imports +from __future__ import print_function +from glob import glob +import platform +import os + +from distutils.core import * +from distutils.command.build import build + +rundir = os.getcwd() +os.chdir(os.path.abspath(os.path.dirname(__file__))) + +if any(i in os.environ for i in ["CC", "CXX"]): + if "CC" not in os.environ: + os.environ['CC'] = os.environ['CXX'] + if "CXX" not in os.environ: + os.environ['CXX'] = os.environ['CC'] + if platform.system() == 'Linux': + os.environ['LDSHARED'] = os.environ['CXX'] + " -shared" + print("Environment specifies CC=%s CXX=%s"%(os.environ['CC'], os.environ['CXX'])) + +intel_compiler = os.getenv('CC', '') in ['icl', 'icpc', 'icc'] +try: + tbb_root = os.environ['TBBROOT'] + print("Using TBBROOT=", tbb_root) +except: + tbb_root = '..' + if not intel_compiler: + print("Warning: TBBROOT env var is not set and Intel's compiler is not used. It might lead\n" + " !!!: to compile/link problems. Source tbbvars.sh/.csh file to set environment") +use_compiler_tbb = intel_compiler and tbb_root == '..' +if use_compiler_tbb: + print("Using Intel TBB from Intel's compiler") +if platform.system() == 'Windows': + if intel_compiler: + os.environ['DISTUTILS_USE_SDK'] = '1' # Enable environment settings in distutils + os.environ['MSSdk'] = '1' + print("Using compiler settings from environment") + tbb_flag = ['/Qtbb'] if use_compiler_tbb else [] + tbb_flag += ['/EHsc'] # for Python 2 + compile_flags = ['/Qstd=c++11'] if intel_compiler else [] +else: + tbb_flag = ['-tbb'] if use_compiler_tbb else [] + compile_flags = ['-std=c++11', '-Wno-unused-variable'] + +_tbb = Extension("tbb._api", ["tbb/api.i"], + include_dirs=[os.path.join(tbb_root, 'include')] if not use_compiler_tbb else [], + swig_opts =['-c++', '-O', '-threads'] + ( # add '-builtin' later + ['-I' + os.path.join(tbb_root, 'include')] if not use_compiler_tbb else []), + extra_compile_args=compile_flags + tbb_flag, + extra_link_args=tbb_flag, + libraries =(['tbb'] if not use_compiler_tbb else []) + + (['irml'] if platform.system() == "Linux" else []), # TODO: why do we need this? + library_dirs=[ rundir, # for custom-builds + os.path.join(tbb_root, 'lib', 'intel64', 'gcc4.4'), # for Linux + os.path.join(tbb_root, 'lib'), # for MacOS + os.path.join(tbb_root, 'lib', 'intel64', 'vc_mt'), # for Windows + ] if not use_compiler_tbb else [], + language ='c++', + ) + + +class TBBBuild(build): + sub_commands = [ # define build order + ('build_ext', build.has_ext_modules), + ('build_py', build.has_pure_modules), + ] + + +setup( name ="TBB", + description ="Python API for Intel TBB", + long_description="Python API to Intel(R) Threading Building Blocks library (Intel(R) TBB) " + "extended with standard Pool implementation and monkey-patching", + url ="https://software.intel.com/en-us/intel-tbb", + author ="Intel Corporation", + author_email="inteltbbdevelopers@intel.com", + license ="Dual license: Apache or Proprietary", + version ="0.1", + classifiers =[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: Plugins', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Other Audience', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'Programming Language :: C++', + 'Topic :: System :: Hardware :: Symmetric Multi-processing', + 'Topic :: Software Development :: Libraries', + ], + keywords='TBB multiprocessing multithreading composable parallelism', + ext_modules=[_tbb], + packages=['tbb'], + py_modules=['TBB'], + cmdclass={'build': TBBBuild} +) diff --git a/src/tbb/python/tbb/__init__.py b/src/tbb/python/tbb/__init__.py new file mode 100644 index 00000000..0fb79fed --- /dev/null +++ b/src/tbb/python/tbb/__init__.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import print_function + +import multiprocessing.pool +import ctypes +import atexit +import sys +import os + +from .api import * +from .api import __all__ as api__all +from .pool import * +from .pool import __all__ as pool__all + +__all__ = ["Monkey", "is_active"] + api__all + pool__all + +__doc__ = """ +Python API for Intel(R) Threading Building Blocks library (Intel(R) TBB) +extended with standard Python's pools implementation and monkey-patching. + +Command-line interface example: +$ python -m tbb $your_script.py +Runs your_script.py in context of tbb.Monkey +""" + +is_active = False +""" Indicates whether TBB context is activated """ + +ipc_enabled = False +""" Indicates whether IPC mode is enabled """ + +libirml = "libirml.so.1" + + +def _test(arg=None): + """Some tests""" + import platform + if platform.system() == "Linux": + ctypes.CDLL(libirml) + assert 256 == os.system("ldd "+_api.__file__+"| grep -E 'libimf|libsvml|libintlc'") + from .test import test + test(arg) + print("done") + + +def tbb_process_pool_worker27(inqueue, outqueue, initializer=None, initargs=(), + maxtasks=None): + from multiprocessing.pool import worker + worker(inqueue, outqueue, initializer, initargs, maxtasks) + if ipc_enabled: + try: + librml = ctypes.CDLL(libirml) + librml.release_resources() + except: + print("Warning: Can not load ", libirml, file=sys.stderr) + + +class TBBProcessPool27(multiprocessing.pool.Pool): + def _repopulate_pool(self): + """Bring the number of pool processes up to the specified number, + for use after reaping workers which have exited. + """ + from multiprocessing.util import debug + + for i in range(self._processes - len(self._pool)): + w = self.Process(target=tbb_process_pool_worker27, + args=(self._inqueue, self._outqueue, + self._initializer, + self._initargs, self._maxtasksperchild) + ) + self._pool.append(w) + w.name = w.name.replace('Process', 'PoolWorker') + w.daemon = True + w.start() + debug('added worker') + + def __del__(self): + self.close() + for p in self._pool: + p.join() + + def __exit__(self, *args): + self.close() + for p in self._pool: + p.join() + + +def tbb_process_pool_worker3(inqueue, outqueue, initializer=None, initargs=(), + maxtasks=None, wrap_exception=False): + from multiprocessing.pool import worker + worker(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception) + if ipc_enabled: + try: + librml = ctypes.CDLL(libirml) + librml.release_resources() + except: + print("Warning: Can not load ", libirml, file=sys.stderr) + + +class TBBProcessPool3(multiprocessing.pool.Pool): + def _repopulate_pool(self): + """Bring the number of pool processes up to the specified number, + for use after reaping workers which have exited. + """ + from multiprocessing.util import debug + + for i in range(self._processes - len(self._pool)): + w = self.Process(target=tbb_process_pool_worker3, + args=(self._inqueue, self._outqueue, + self._initializer, + self._initargs, self._maxtasksperchild, + self._wrap_exception) + ) + self._pool.append(w) + w.name = w.name.replace('Process', 'PoolWorker') + w.daemon = True + w.start() + debug('added worker') + + def __del__(self): + self.close() + for p in self._pool: + p.join() + + def __exit__(self, *args): + self.close() + for p in self._pool: + p.join() + + +class Monkey: + """ + Context manager which replaces standard multiprocessing.pool + implementations with tbb.pool using monkey-patching. It also enables TBB + threading for Intel(R) Math Kernel Library (Intel(R) MKL). For example: + + with tbb.Monkey(): + run_my_numpy_code() + + It allows multiple parallel tasks to be executed on the same thread pool + and coordinate number of threads across multiple processes thus avoiding + overheads from oversubscription. + """ + _items = {} + _modules = {} + + def __init__(self, max_num_threads=None, benchmark=False): + """ + Create context manager for running under TBB scheduler. + :param max_num_threads: if specified, limits maximal number of threads + :param benchmark: if specified, blocks in initialization until requested number of threads are ready + """ + if max_num_threads: + self.ctl = global_control(global_control.max_allowed_parallelism, int(max_num_threads)) + if benchmark: + if not max_num_threads: + max_num_threads = default_num_threads() + from .api import _concurrency_barrier + _concurrency_barrier(int(max_num_threads)) + + def _patch(self, class_name, module_name, obj): + m = self._modules[class_name] = __import__(module_name, globals(), + locals(), [class_name]) + if m == None: + return + oldattr = getattr(m, class_name, None) + if oldattr == None: + self._modules[class_name] = None + return + self._items[class_name] = oldattr + setattr(m, class_name, obj) + + def __enter__(self): + global is_active + assert is_active == False, "tbb.Monkey does not support nesting yet" + is_active = True + self.env_mkl = os.getenv('MKL_THREADING_LAYER') + os.environ['MKL_THREADING_LAYER'] = 'TBB' + self.env_numba = os.getenv('NUMBA_THREADING_LAYER') + os.environ['NUMBA_THREADING_LAYER'] = 'TBB' + + if ipc_enabled: + if sys.version_info.major == 2 and sys.version_info.minor >= 7: + self._patch("Pool", "multiprocessing.pool", TBBProcessPool27) + elif sys.version_info.major == 3 and sys.version_info.minor >= 5: + self._patch("Pool", "multiprocessing.pool", TBBProcessPool3) + self._patch("ThreadPool", "multiprocessing.pool", Pool) + return self + + def __exit__(self, exc_type, exc_value, traceback): + global is_active + assert is_active == True, "modified?" + is_active = False + if self.env_mkl is None: + del os.environ['MKL_THREADING_LAYER'] + else: + os.environ['MKL_THREADING_LAYER'] = self.env_mkl + if self.env_numba is None: + del os.environ['NUMBA_THREADING_LAYER'] + else: + os.environ['NUMBA_THREADING_LAYER'] = self.env_numba + for name in self._items.keys(): + setattr(self._modules[name], name, self._items[name]) + + +def init_sem_name(): + try: + librml = ctypes.CDLL(libirml) + librml.set_active_sem_name() + librml.set_stop_sem_name() + except Exception as e: + print("Warning: Can not initialize name of shared semaphores:", e, + file=sys.stderr) + + +def tbb_atexit(): + if ipc_enabled: + try: + librml = ctypes.CDLL(libirml) + librml.release_semaphores() + except: + print("Warning: Can not release shared semaphores", + file=sys.stderr) + + +def _main(): + # Run the module specified as the next command line argument + # python -m TBB user_app.py + global ipc_enabled + + import platform + import argparse + parser = argparse.ArgumentParser(prog="python -m tbb", description=""" + Run your Python script in context of tbb.Monkey, which + replaces standard Python pools and threading layer of + Intel(R) Math Kernel Library by implementation based on + Intel(R) Threading Building Blocks. It enables multiple parallel + tasks to be executed on the same thread pool and coordinate + number of threads across multiple processes thus avoiding + overheads from oversubscription. + """, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + if platform.system() == "Linux": + parser.add_argument('--ipc', action='store_true', + help="Enable inter-process (IPC) coordination between Intel TBB schedulers") + parser.add_argument('-a', '--allocator', action='store_true', + help="Enable Intel TBB scalable allocator as a replacement for standard memory allocator") + parser.add_argument('--allocator-huge-pages', action='store_true', + help="Enable huge pages for Intel TBB allocator (implies: -a)") + parser.add_argument('-p', '--max-num-threads', default=default_num_threads(), type=int, + help="Initialize Intel TBB with P max number of threads per process", metavar='P') + parser.add_argument('-b', '--benchmark', action='store_true', + help="Block Intel TBB initialization until all the threads are created before continue the script. " + "This is necessary for performance benchmarks that want to exclude lazy scheduler initialization effects from the measurements") + parser.add_argument('-v', '--verbose', action='store_true', + help="Request verbose and version information") + parser.add_argument('-m', action='store_true', dest='module', + help="Executes following as a module") + parser.add_argument('name', help="Script or module name") + parser.add_argument('args', nargs=argparse.REMAINDER, + help="Command line arguments") + args = parser.parse_args() + + if args.verbose: + os.environ["TBB_VERSION"] = "1" + if platform.system() == "Linux": + if args.allocator_huge_pages: + args.allocator = True + if args.allocator and not os.environ.get("_TBB_MALLOC_PRELOAD"): + libtbbmalloc_lib = 'libtbbmalloc_proxy.so.2' + ld_preload = 'LD_PRELOAD' + os.environ["_TBB_MALLOC_PRELOAD"] = "1" + preload_list = filter(None, os.environ.get(ld_preload, "").split(':')) + if libtbbmalloc_lib in preload_list: + print('Info:', ld_preload, "contains", libtbbmalloc_lib, "already\n") + else: + os.environ[ld_preload] = ':'.join([libtbbmalloc_lib] + list(preload_list)) + + if args.allocator_huge_pages: + assert platform.system() == "Linux" + try: + with open('/proc/sys/vm/nr_hugepages', 'r') as f: + pages = int(f.read()) + if pages == 0: + print("TBB: Pre-allocated huge pages are not currently reserved in the system. To reserve, run e.g.:\n" + "\tsudo sh -c 'echo 2000 > /proc/sys/vm/nr_hugepages'") + os.environ["TBB_MALLOC_USE_HUGE_PAGES"] = "1" + except: + print("TBB: Failed to read number of pages from /proc/sys/vm/nr_hugepages\n" + "\tIs the Linux kernel configured with the huge pages feature?") + sys.exit(1) + + os.execl(sys.executable, sys.executable, '-m', 'tbb', *sys.argv[1:]) + assert False, "Re-execution failed" + + sys.argv = [args.name] + args.args + ipc_enabled = platform.system() == "Linux" and args.ipc + os.environ["IPC_ENABLE"] = "1" if ipc_enabled else "0" + if ipc_enabled: + atexit.register(tbb_atexit) + init_sem_name() + if not os.environ.get("KMP_BLOCKTIME"): # TODO move + os.environ["KMP_BLOCKTIME"] = "0" + if '_' + args.name in globals(): + return globals()['_' + args.name](*args.args) + else: + import runpy + runf = runpy.run_module if args.module else runpy.run_path + with Monkey(max_num_threads=args.max_num_threads, benchmark=args.benchmark): + runf(args.name, run_name='__main__') diff --git a/src/tbb/python/tbb/__main__.py b/src/tbb/python/tbb/__main__.py new file mode 100644 index 00000000..35d005e1 --- /dev/null +++ b/src/tbb/python/tbb/__main__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import _main +from sys import exit +exit(_main()) diff --git a/src/tbb/python/tbb/api.i b/src/tbb/python/tbb/api.i new file mode 100644 index 00000000..56c8e7aa --- /dev/null +++ b/src/tbb/python/tbb/api.i @@ -0,0 +1,175 @@ +%pythonbegin %{ +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__all__ = ["task_arena", "task_group", "task_scheduler_init", "global_control", "default_num_threads"] +%} +%begin %{ +/* Defines Python wrappers for Intel(R) Threading Building Blocks (Intel TBB).*/ +%} +%module api + +#if SWIG_VERSION < 0x030001 +#error SWIG version 3.0.6 or newer is required for correct functioning +#endif + +%{ +#define TBB_PREVIEW_WAITING_FOR_WORKERS 1 +#include +#include +#if TBB_IMPLEMENT_CPP0X +namespace std { using tbb::mutex; } +#define unique_ptr auto_ptr +#else +#include +#include +#include +#endif +using namespace tbb; + +class PyCaller : public swig::SwigPtr_PyObject { +public: + // icpc 2013 does not support simple using SwigPtr_PyObject::SwigPtr_PyObject; + PyCaller(const PyCaller& s) : SwigPtr_PyObject(s) {} + PyCaller(PyObject *p, bool initial = true) : SwigPtr_PyObject(p, initial) {} + + void operator()() const { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + PyObject* r = PyObject_CallFunctionObjArgs((PyObject*)*this, NULL); + if(r) Py_DECREF(r); + SWIG_PYTHON_THREAD_END_BLOCK; + } +}; + +struct ArenaPyCaller { + task_arena *my_arena; + PyObject *my_callable; + ArenaPyCaller(task_arena *a, PyObject *c) : my_arena(a), my_callable(c) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_XINCREF(c); + SWIG_PYTHON_THREAD_END_BLOCK; + } + void operator()() const { + my_arena->execute(PyCaller(my_callable, false)); + } +}; + +struct barrier_data { + std::condition_variable event; + std::mutex m; + int worker_threads, full_threads; +}; + +class barrier_task : public tbb::task { + barrier_data &b; +public: + barrier_task(barrier_data &d) : b(d) {} + /*override*/ tbb::task *execute() { + std::unique_lock lock(b.m); + if(++b.worker_threads >= b.full_threads) + b.event.notify_all(); + else while(b.worker_threads < b.full_threads) + b.event.wait(lock); + return NULL; + } +}; + +void _concurrency_barrier(int threads = tbb::task_scheduler_init::automatic) { + if(threads == task_scheduler_init::automatic) + threads = task_scheduler_init::default_num_threads(); + if(threads < 2) + return; + std::unique_ptr g( + (global_control::active_value(global_control::max_allowed_parallelism) < unsigned(threads))? + new global_control(global_control::max_allowed_parallelism, threads) : NULL); + barrier_data b; + b.worker_threads = 0; + b.full_threads = threads-1; + for(int i = 0; i < b.full_threads; i++) + tbb::task::enqueue( *new( tbb::task::allocate_root() ) barrier_task(b) ); + std::unique_lock lock(b.m); + b.event.wait(lock); +}; + +%} + +void _concurrency_barrier(int threads = tbb::task_scheduler_init::automatic); + +namespace tbb { + class task_scheduler_init { + public: + //! Typedef for number of threads that is automatic. + static const int automatic = -1; + //! Argument to initialize() or constructor that causes initialization to be deferred. + static const int deferred = -2; + task_scheduler_init( int max_threads=automatic, + size_t thread_stack_size=0 ); + ~task_scheduler_init(); + void initialize( int max_threads=automatic ); + void terminate(); + static int default_num_threads(); + bool is_active() const; + void blocking_terminate(); + }; + + class task_arena { + public: + static const int automatic = -1; + static int current_thread_index(); + task_arena(int max_concurrency = automatic, unsigned reserved_for_masters = 1); + task_arena(const task_arena &s); + ~task_arena(); + void initialize(); + void initialize(int max_concurrency, unsigned reserved_for_masters = 1); + void terminate(); + bool is_active(); + %extend { + void enqueue( PyObject *c ) { $self->enqueue(PyCaller(c)); } + void execute( PyObject *c ) { $self->execute(PyCaller(c)); } + }; + }; + + class task_group { + public: + task_group(); + ~task_group(); + void wait(); + bool is_canceling(); + void cancel(); + %extend { + void run( PyObject *c ) { $self->run(PyCaller(c)); } + void run( PyObject *c, task_arena *a ) { $self->run(ArenaPyCaller(a, c)); } + }; + }; + + class global_control { + public: + enum parameter { + max_allowed_parallelism, + thread_stack_size, + parameter_max // insert new parameters above this point + }; + global_control(parameter param, size_t value); + ~global_control(); + static size_t active_value(parameter param); + }; + +} // tbb + +// Additional definitions for Python part of the module +%pythoncode %{ +default_num_threads = task_scheduler_init_default_num_threads +%} diff --git a/src/tbb/python/tbb/pool.py b/src/tbb/python/tbb/pool.py new file mode 100644 index 00000000..3a5da11c --- /dev/null +++ b/src/tbb/python/tbb/pool.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based on the software developed by: +# Copyright (c) 2008,2016 david decotigny (Pool of threads) +# Copyright (c) 2006-2008, R Oudkerk (multiprocessing.Pool) +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# @brief Python Pool implementation based on TBB with monkey-patching +# +# See http://docs.python.org/dev/library/multiprocessing.html +# Differences: added imap_async and imap_unordered_async, and terminate() +# has to be called explicitly (it's not registered by atexit). +# +# The general idea is that we submit works to a workqueue, either as +# single Jobs (one function to call), or JobSequences (batch of +# Jobs). Each Job is associated with an ApplyResult object which has 2 +# states: waiting for the Job to complete, or Ready. Instead of +# waiting for the jobs to finish, we wait for their ApplyResult object +# to become ready: an event mechanism is used for that. +# When we apply a function to several arguments in "parallel", we need +# a way to wait for all/part of the Jobs to be processed: that's what +# "collectors" are for; they group and wait for a set of ApplyResult +# objects. Once a collector is ready to be used, we can use a +# CollectorIterator to iterate over the result values it's collecting. +# +# The methods of a Pool object use all these concepts and expose +# them to their caller in a very simple way. + +import sys +import threading +import traceback +from .api import * + +__all__ = ["Pool", "TimeoutError"] +__doc__ = """ +Standard Python Pool implementation based on Python API +for Intel(R) Threading Building Blocks library (Intel(R) TBB) +""" + + +class TimeoutError(Exception): + """Raised when a result is not available within the given timeout""" + pass + + +class Pool(object): + """ + The Pool class provides standard multiprocessing.Pool interface + which is mapped onto Intel(R) TBB tasks executing in its thread pool + """ + + def __init__(self, nworkers=0, name="Pool"): + """ + \param nworkers (integer) number of worker threads to start + \param name (string) prefix for the worker threads' name + """ + self._closed = False + self._tasks = task_group() + self._pool = [None,]*default_num_threads() # Dask asks for len(_pool) + + def apply(self, func, args=(), kwds=dict()): + """Equivalent of the apply() builtin function. It blocks till + the result is ready.""" + return self.apply_async(func, args, kwds).get() + + def map(self, func, iterable, chunksize=None): + """A parallel equivalent of the map() builtin function. It + blocks till the result is ready. + + This method chops the iterable into a number of chunks which + it submits to the process pool as separate tasks. The + (approximate) size of these chunks can be specified by setting + chunksize to a positive integer.""" + return self.map_async(func, iterable, chunksize).get() + + def imap(self, func, iterable, chunksize=1): + """ + An equivalent of itertools.imap(). + + The chunksize argument is the same as the one used by the + map() method. For very long iterables using a large value for + chunksize can make the job complete much faster than + using the default value of 1. + + Also if chunksize is 1 then the next() method of the iterator + returned by the imap() method has an optional timeout + parameter: next(timeout) will raise processing.TimeoutError if + the result cannot be returned within timeout seconds. + """ + collector = OrderedResultCollector(as_iterator=True) + self._create_sequences(func, iterable, chunksize, collector) + return iter(collector) + + def imap_unordered(self, func, iterable, chunksize=1): + """The same as imap() except that the ordering of the results + from the returned iterator should be considered + arbitrary. (Only when there is only one worker process is the + order guaranteed to be "correct".)""" + collector = UnorderedResultCollector() + self._create_sequences(func, iterable, chunksize, collector) + return iter(collector) + + def apply_async(self, func, args=(), kwds=dict(), callback=None): + """A variant of the apply() method which returns an + ApplyResult object. + + If callback is specified then it should be a callable which + accepts a single argument. When the result becomes ready, + callback is applied to it (unless the call failed). callback + should complete immediately since otherwise the thread which + handles the results will get blocked.""" + assert not self._closed # No lock here. We assume it's atomic... + apply_result = ApplyResult(callback=callback) + job = Job(func, args, kwds, apply_result) + self._tasks.run(job) + return apply_result + + def map_async(self, func, iterable, chunksize=None, callback=None): + """A variant of the map() method which returns a ApplyResult + object. + + If callback is specified then it should be a callable which + accepts a single argument. When the result becomes ready + callback is applied to it (unless the call failed). callback + should complete immediately since otherwise the thread which + handles the results will get blocked.""" + apply_result = ApplyResult(callback=callback) + collector = OrderedResultCollector(apply_result, as_iterator=False) + if not self._create_sequences(func, iterable, chunksize, collector): + apply_result._set_value([]) + return apply_result + + def imap_async(self, func, iterable, chunksize=None, callback=None): + """A variant of the imap() method which returns an ApplyResult + object that provides an iterator (next method(timeout) + available). + + If callback is specified then it should be a callable which + accepts a single argument. When the resulting iterator becomes + ready, callback is applied to it (unless the call + failed). callback should complete immediately since otherwise + the thread which handles the results will get blocked.""" + apply_result = ApplyResult(callback=callback) + collector = OrderedResultCollector(apply_result, as_iterator=True) + if not self._create_sequences(func, iterable, chunksize, collector): + apply_result._set_value(iter([])) + return apply_result + + def imap_unordered_async(self, func, iterable, chunksize=None, + callback=None): + """A variant of the imap_unordered() method which returns an + ApplyResult object that provides an iterator (next + method(timeout) available). + + If callback is specified then it should be a callable which + accepts a single argument. When the resulting iterator becomes + ready, callback is applied to it (unless the call + failed). callback should complete immediately since otherwise + the thread which handles the results will get blocked.""" + apply_result = ApplyResult(callback=callback) + collector = UnorderedResultCollector(apply_result) + if not self._create_sequences(func, iterable, chunksize, collector): + apply_result._set_value(iter([])) + return apply_result + + def close(self): + """Prevents any more tasks from being submitted to the + pool. Once all the tasks have been completed the worker + processes will exit.""" + # No lock here. We assume it's sufficiently atomic... + self._closed = True + + def terminate(self): + """Stops the worker processes immediately without completing + outstanding work. When the pool object is garbage collected + terminate() will be called immediately.""" + self.close() + self._tasks.cancel() + + def join(self): + """Wait for the worker processes to exit. One must call + close() or terminate() before using join().""" + self._tasks.wait() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.join() + + def __del__(self): + self.terminate() + self.join() + + def _create_sequences(self, func, iterable, chunksize, collector): + """ + Create callable objects to process and pushes them on the + work queue. Each work unit is meant to process a slice of + iterable of size chunksize. If collector is specified, then + the ApplyResult objects associated with the jobs will notify + collector when their result becomes ready. + + \return the list callable objects (basically: JobSequences) + pushed onto the work queue + """ + assert not self._closed # No lock here. We assume it's atomic... + it_ = iter(iterable) + exit_loop = False + sequences = [] + while not exit_loop: + seq = [] + for _ in range(chunksize or 1): + try: + arg = next(it_) + except StopIteration: + exit_loop = True + break + apply_result = ApplyResult(collector) + job = Job(func, (arg,), {}, apply_result) + seq.append(job) + if seq: + sequences.append(JobSequence(seq)) + for t in sequences: + self._tasks.run(t) + return sequences + + +class Job: + """A work unit that corresponds to the execution of a single function""" + + def __init__(self, func, args, kwds, apply_result): + """ + \param func/args/kwds used to call the function + \param apply_result ApplyResult object that holds the result + of the function call + """ + self._func = func + self._args = args + self._kwds = kwds + self._result = apply_result + + def __call__(self): + """ + Call the function with the args/kwds and tell the ApplyResult + that its result is ready. Correctly handles the exceptions + happening during the execution of the function + """ + try: + result = self._func(*self._args, **self._kwds) + except: + self._result._set_exception() + else: + self._result._set_value(result) + + +class JobSequence: + """A work unit that corresponds to the processing of a continuous + sequence of Job objects""" + + def __init__(self, jobs): + self._jobs = jobs + + def __call__(self): + """ + Call all the Job objects that have been specified + """ + for job in self._jobs: + job() + + +class ApplyResult(object): + """An object associated with a Job object that holds its result: + it's available during the whole life the Job and after, even when + the Job didn't process yet. It's possible to use this object to + wait for the result/exception of the job to be available. + + The result objects returns by the Pool::*_async() methods are of + this type""" + + def __init__(self, collector=None, callback=None): + """ + \param collector when not None, the notify_ready() method of + the collector will be called when the result from the Job is + ready + \param callback when not None, function to call when the + result becomes available (this is the parameter passed to the + Pool::*_async() methods. + """ + self._success = False + self._event = threading.Event() + self._data = None + self._collector = None + self._callback = callback + + if collector is not None: + collector.register_result(self) + self._collector = collector + + def get(self, timeout=None): + """ + Returns the result when it arrives. If timeout is not None and + the result does not arrive within timeout seconds then + TimeoutError is raised. If the remote call raised an exception + then that exception will be reraised by get(). + """ + if not self.wait(timeout): + raise TimeoutError("Result not available within %fs" % timeout) + if self._success: + return self._data + if sys.version_info[0] == 3: + raise self._data[0](self._data[1]).with_traceback(self._data[2]) + else: + exec("raise self._data[0], self._data[1], self._data[2]") + + def wait(self, timeout=None): + """Waits until the result is available or until timeout + seconds pass.""" + self._event.wait(timeout) + return self._event.isSet() + + def ready(self): + """Returns whether the call has completed.""" + return self._event.isSet() + + def successful(self): + """Returns whether the call completed without raising an + exception. Will raise AssertionError if the result is not + ready.""" + assert self.ready() + return self._success + + def _set_value(self, value): + """Called by a Job object to tell the result is ready, and + provides the value of this result. The object will become + ready and successful. The collector's notify_ready() method + will be called, and the callback method too""" + assert not self.ready() + self._data = value + self._success = True + self._event.set() + if self._collector is not None: + self._collector.notify_ready(self) + if self._callback is not None: + try: + self._callback(value) + except: + traceback.print_exc() + + def _set_exception(self): + """Called by a Job object to tell that an exception occurred + during the processing of the function. The object will become + ready but not successful. The collector's notify_ready() + method will be called, but NOT the callback method""" + # traceback.print_exc() + assert not self.ready() + self._data = sys.exc_info() + self._success = False + self._event.set() + if self._collector is not None: + self._collector.notify_ready(self) + + +class AbstractResultCollector(object): + """ABC to define the interface of a ResultCollector object. It is + basically an object which knows whuich results it's waiting for, + and which is able to get notify when they get available. It is + also able to provide an iterator over the results when they are + available""" + + def __init__(self, to_notify): + """ + \param to_notify ApplyResult object to notify when all the + results we're waiting for become available. Can be None. + """ + self._to_notify = to_notify + + def register_result(self, apply_result): + """Used to identify which results we're waiting for. Will + always be called BEFORE the Jobs get submitted to the work + queue, and BEFORE the __iter__ and _get_result() methods can + be called + \param apply_result ApplyResult object to add in our collection + """ + raise NotImplementedError("Children classes must implement it") + + def notify_ready(self, apply_result): + """Called by the ApplyResult object (already registered via + register_result()) that it is now ready (ie. the Job's result + is available or an exception has been raised). + \param apply_result ApplyResult object telling us that the job + has been processed + """ + raise NotImplementedError("Children classes must implement it") + + def _get_result(self, idx, timeout=None): + """Called by the CollectorIterator object to retrieve the + result's values one after another (order defined by the + implementation) + \param idx The index of the result we want, wrt collector's order + \param timeout integer telling how long to wait (in seconds) + for the result at index idx to be available, or None (wait + forever) + """ + raise NotImplementedError("Children classes must implement it") + + def __iter__(self): + """Return a new CollectorIterator object for this collector""" + return CollectorIterator(self) + + +class CollectorIterator(object): + """An iterator that allows to iterate over the result values + available in the given collector object. Equipped with an extended + next() method accepting a timeout argument. Created by the + AbstractResultCollector::__iter__() method""" + + def __init__(self, collector): + """\param AbstractResultCollector instance""" + self._collector = collector + self._idx = 0 + + def __iter__(self): + return self + + def next(self, timeout=None): + """Return the next result value in the sequence. Raise + StopIteration at the end. Can raise the exception raised by + the Job""" + try: + apply_result = self._collector._get_result(self._idx, timeout) + except IndexError: + # Reset for next time + self._idx = 0 + raise StopIteration + except: + self._idx = 0 + raise + self._idx += 1 + assert apply_result.ready() + return apply_result.get(0) + + def __next__(self): + return self.next() + + +class UnorderedResultCollector(AbstractResultCollector): + """An AbstractResultCollector implementation that collects the + values of the ApplyResult objects in the order they become ready. The + CollectorIterator object returned by __iter__() will iterate over + them in the order they become ready""" + + def __init__(self, to_notify=None): + """ + \param to_notify ApplyResult object to notify when all the + results we're waiting for become available. Can be None. + """ + AbstractResultCollector.__init__(self, to_notify) + self._cond = threading.Condition() + self._collection = [] + self._expected = 0 + + def register_result(self, apply_result): + """Used to identify which results we're waiting for. Will + always be called BEFORE the Jobs get submitted to the work + queue, and BEFORE the __iter__ and _get_result() methods can + be called + \param apply_result ApplyResult object to add in our collection + """ + self._expected += 1 + + def _get_result(self, idx, timeout=None): + """Called by the CollectorIterator object to retrieve the + result's values one after another, in the order the results have + become available. + \param idx The index of the result we want, wrt collector's order + \param timeout integer telling how long to wait (in seconds) + for the result at index idx to be available, or None (wait + forever) + """ + self._cond.acquire() + try: + if idx >= self._expected: + raise IndexError + elif idx < len(self._collection): + return self._collection[idx] + elif idx != len(self._collection): + # Violation of the sequence protocol + raise IndexError() + else: + self._cond.wait(timeout=timeout) + try: + return self._collection[idx] + except IndexError: + # Still not added ! + raise TimeoutError("Timeout while waiting for results") + finally: + self._cond.release() + + def notify_ready(self, apply_result=None): + """Called by the ApplyResult object (already registered via + register_result()) that it is now ready (ie. the Job's result + is available or an exception has been raised). + \param apply_result ApplyResult object telling us that the job + has been processed + """ + first_item = False + self._cond.acquire() + try: + self._collection.append(apply_result) + first_item = (len(self._collection) == 1) + + self._cond.notifyAll() + finally: + self._cond.release() + + if first_item and self._to_notify is not None: + self._to_notify._set_value(iter(self)) + + +class OrderedResultCollector(AbstractResultCollector): + """An AbstractResultCollector implementation that collects the + values of the ApplyResult objects in the order they have been + submitted. The CollectorIterator object returned by __iter__() + will iterate over them in the order they have been submitted""" + + def __init__(self, to_notify=None, as_iterator=True): + """ + \param to_notify ApplyResult object to notify when all the + results we're waiting for become available. Can be None. + \param as_iterator boolean telling whether the result value + set on to_notify should be an iterator (available as soon as 1 + result arrived) or a list (available only after the last + result arrived) + """ + AbstractResultCollector.__init__(self, to_notify) + self._results = [] + self._lock = threading.Lock() + self._remaining = 0 + self._as_iterator = as_iterator + + def register_result(self, apply_result): + """Used to identify which results we're waiting for. Will + always be called BEFORE the Jobs get submitted to the work + queue, and BEFORE the __iter__ and _get_result() methods can + be called + \param apply_result ApplyResult object to add in our collection + """ + self._results.append(apply_result) + self._remaining += 1 + + def _get_result(self, idx, timeout=None): + """Called by the CollectorIterator object to retrieve the + result's values one after another (order defined by the + implementation) + \param idx The index of the result we want, wrt collector's order + \param timeout integer telling how long to wait (in seconds) + for the result at index idx to be available, or None (wait + forever) + """ + res = self._results[idx] + res.wait(timeout) + return res + + def notify_ready(self, apply_result): + """Called by the ApplyResult object (already registered via + register_result()) that it is now ready (ie. the Job's result + is available or an exception has been raised). + \param apply_result ApplyResult object telling us that the job + has been processed + """ + got_first = False + got_last = False + self._lock.acquire() + try: + assert self._remaining > 0 + got_first = (len(self._results) == self._remaining) + self._remaining -= 1 + got_last = (self._remaining == 0) + finally: + self._lock.release() + + if self._to_notify is not None: + if self._as_iterator and got_first: + self._to_notify._set_value(iter(self)) + elif not self._as_iterator and got_last: + try: + lst = [r.get(0) for r in self._results] + except: + self._to_notify._set_exception() + else: + self._to_notify._set_value(lst) diff --git a/src/tbb/python/tbb/test.py b/src/tbb/python/tbb/test.py new file mode 100644 index 00000000..c95e4cc9 --- /dev/null +++ b/src/tbb/python/tbb/test.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016-2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based on the software developed by: +# Copyright (c) 2008,2016 david decotigny (Pool of threads) +# Copyright (c) 2006-2008, R Oudkerk (multiprocessing.Pool) +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +from __future__ import print_function +import time +import threading + +from .api import * +from .pool import * + + +def test(arg=None): + if arg == "-v": + def say(*x): + print(*x) + else: + def say(*x): + pass + say("Start Pool testing") + + get_tid = lambda: threading.current_thread().ident + + def return42(): + return 42 + + def f(x): + return x * x + + def work(mseconds): + res = str(mseconds) + if mseconds < 0: + mseconds = -mseconds + say("[%d] Start to work for %fms..." % (get_tid(), mseconds*10)) + time.sleep(mseconds/100.) + say("[%d] Work done (%fms)." % (get_tid(), mseconds*10)) + return res + + ### Test copy/pasted from multiprocessing + pool = Pool(4) # start worker threads + + # edge cases + assert pool.map(return42, []) == [] + assert pool.apply_async(return42, []).get() == 42 + assert pool.apply(return42, []) == 42 + assert list(pool.imap(return42, iter([]))) == [] + assert list(pool.imap_unordered(return42, iter([]))) == [] + assert pool.map_async(return42, []).get() == [] + assert list(pool.imap_async(return42, iter([])).get()) == [] + assert list(pool.imap_unordered_async(return42, iter([])).get()) == [] + + # basic tests + result = pool.apply_async(f, (10,)) # evaluate "f(10)" asynchronously + assert result.get(timeout=1) == 100 # ... unless slow computer + assert list(pool.map(f, range(10))) == list(map(f, range(10))) + it = pool.imap(f, range(10)) + assert next(it) == 0 + assert next(it) == 1 + assert next(it) == 4 + + # Test apply_sync exceptions + result = pool.apply_async(time.sleep, (3,)) + try: + say(result.get(timeout=1)) # raises `TimeoutError` + except TimeoutError: + say("Good. Got expected timeout exception.") + else: + assert False, "Expected exception !" + assert result.get() is None # sleep() returns None + + def cb(s): + say("Result ready: %s" % s) + + # Test imap() + assert list(pool.imap(work, range(10, 3, -1), chunksize=4)) == list(map( + str, range(10, 3, -1))) + + # Test imap_unordered() + assert sorted(pool.imap_unordered(work, range(10, 3, -1))) == sorted(map( + str, range(10, 3, -1))) + + # Test map_async() + result = pool.map_async(work, range(10), callback=cb) + try: + result.get(timeout=0.01) # raises `TimeoutError` + except TimeoutError: + say("Good. Got expected timeout exception.") + else: + assert False, "Expected exception !" + say(result.get()) + + # Test imap_async() + result = pool.imap_async(work, range(3, 10), callback=cb) + try: + result.get(timeout=0.01) # raises `TimeoutError` + except TimeoutError: + say("Good. Got expected timeout exception.") + else: + assert False, "Expected exception !" + for i in result.get(): + say("Item:", i) + say("### Loop again:") + for i in result.get(): + say("Item2:", i) + + # Test imap_unordered_async() + result = pool.imap_unordered_async(work, range(10, 3, -1), callback=cb) + try: + say(result.get(timeout=0.01)) # raises `TimeoutError` + except TimeoutError: + say("Good. Got expected timeout exception.") + else: + assert False, "Expected exception !" + for i in result.get(): + say("Item1:", i) + for i in result.get(): + say("Item2:", i) + r = result.get() + for i in r: + say("Item3:", i) + for i in r: + say("Item4:", i) + for i in r: + say("Item5:", i) + + # + # The case for the exceptions + # + + # Exceptions in imap_unordered_async() + result = pool.imap_unordered_async(work, range(2, -10, -1), callback=cb) + time.sleep(3) + try: + for i in result.get(): + say("Got item:", i) + except (IOError, ValueError): + say("Good. Got expected exception") + + # Exceptions in imap_async() + result = pool.imap_async(work, range(2, -10, -1), callback=cb) + time.sleep(3) + try: + for i in result.get(): + say("Got item:", i) + except (IOError, ValueError): + say("Good. Got expected exception") + + # Stop the test: need to stop the pool !!! + pool.terminate() + pool.join() + + diff --git a/src/tbb/src/Makefile b/src/tbb/src/Makefile index 195304d5..cd82fe3f 100644 --- a/src/tbb/src/Makefile +++ b/src/tbb/src/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# tbb_root?=.. examples_root:=$(tbb_root)/examples @@ -94,8 +90,8 @@ else @$(MAKE) -C "$(work_dir)_$(cfg)" -r -f $(tbb_root)/build/Makefile.test $@ endif -python_%: tbb_release - bash -c ". $(work_dir)_release$(SLASH)tbbvars.sh && $(MAKE) -rC '$(full_tbb_root)/python' CXX=$(compiler) $(subst python_,,$@)" +python_%: mkdir_release + $(MAKE) -C "$(work_dir)_release" -rf $(tbb_root)/python/Makefile $(subst python_,,$@) .PHONY: test_release test_debug test_release_no_depends test_debug_no_depends .PHONY: tbb_release tbb_debug tbb_test_release tbb_test_debug tbb_test_release_no_depends tbb_test_debug_no_depends diff --git a/src/tbb/src/index.html b/src/tbb/src/index.html index e828987c..43d7e831 100644 --- a/src/tbb/src/index.html +++ b/src/tbb/src/index.html @@ -66,7 +66,7 @@

Files

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/old/concurrent_queue_v2.cpp b/src/tbb/src/old/concurrent_queue_v2.cpp index 1be9dc7f..096561c7 100644 --- a/src/tbb/src/old/concurrent_queue_v2.cpp +++ b/src/tbb/src/old/concurrent_queue_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "concurrent_queue_v2.h" @@ -32,8 +28,6 @@ #define RECORD_EVENTS 0 -using namespace std; - namespace tbb { namespace internal { @@ -210,7 +204,7 @@ concurrent_queue_base::concurrent_queue_base( size_t item_sz ) { __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset((void*) my_rep, 0, sizeof(concurrent_queue_rep)); + std::memset(static_cast(my_rep),0,sizeof(concurrent_queue_rep)); this->item_size = item_sz; } diff --git a/src/tbb/src/old/concurrent_queue_v2.h b/src/tbb/src/old/concurrent_queue_v2.h index 54dd8223..a76ae1c0 100644 --- a/src/tbb/src/old/concurrent_queue_v2.h +++ b/src/tbb/src/old/concurrent_queue_v2.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_queue_H diff --git a/src/tbb/src/old/concurrent_vector_v2.cpp b/src/tbb/src/old/concurrent_vector_v2.cpp index d31b5b3a..4fe39953 100644 --- a/src/tbb/src/old/concurrent_vector_v2.cpp +++ b/src/tbb/src/old/concurrent_vector_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "concurrent_vector_v2.h" @@ -71,7 +67,7 @@ class concurrent_vector_base::helper { void concurrent_vector_base::helper::extend_segment( concurrent_vector_base& v ) { const size_t pointers_per_long_segment = sizeof(void*)==4 ? 32 : 64; segment_t* s = (segment_t*)NFS_Allocate( pointers_per_long_segment, sizeof(segment_t), NULL ); - std::memset( (void*) s, 0, pointers_per_long_segment*sizeof(segment_t) ); + std::memset( static_cast(s), 0, pointers_per_long_segment*sizeof(segment_t) ); // If other threads are trying to set pointers in the short segment, wait for them to finish their // assignments before we copy the short segment to the long segment. atomic_backoff backoff; diff --git a/src/tbb/src/old/concurrent_vector_v2.h b/src/tbb/src/old/concurrent_vector_v2.h index 6dcae595..55398734 100644 --- a/src/tbb/src/old/concurrent_vector_v2.h +++ b/src/tbb/src/old/concurrent_vector_v2.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_vector_H diff --git a/src/tbb/src/old/spin_rw_mutex_v2.cpp b/src/tbb/src/old/spin_rw_mutex_v2.cpp index 02cf633a..6308c9d8 100644 --- a/src/tbb/src/old/spin_rw_mutex_v2.cpp +++ b/src/tbb/src/old/spin_rw_mutex_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "spin_rw_mutex_v2.h" diff --git a/src/tbb/src/old/spin_rw_mutex_v2.h b/src/tbb/src/old/spin_rw_mutex_v2.h index 12e097d6..952f4a11 100644 --- a/src/tbb/src/old/spin_rw_mutex_v2.h +++ b/src/tbb/src/old/spin_rw_mutex_v2.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_spin_rw_mutex_H diff --git a/src/tbb/src/old/task_v2.cpp b/src/tbb/src/old/task_v2.cpp index 9b569c3b..073c6789 100644 --- a/src/tbb/src/old/task_v2.cpp +++ b/src/tbb/src/old/task_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* This compilation unit provides definition of task::destroy( task& ) diff --git a/src/tbb/src/old/test_concurrent_queue_v2.cpp b/src/tbb/src/old/test_concurrent_queue_v2.cpp index b0bb56ba..69d29e03 100644 --- a/src/tbb/src/old/test_concurrent_queue_v2.cpp +++ b/src/tbb/src/old/test_concurrent_queue_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "old/concurrent_queue_v2.h" diff --git a/src/tbb/src/old/test_concurrent_vector_v2.cpp b/src/tbb/src/old/test_concurrent_vector_v2.cpp index 1d0fd2a7..acb198f5 100644 --- a/src/tbb/src/old/test_concurrent_vector_v2.cpp +++ b/src/tbb/src/old/test_concurrent_vector_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "concurrent_vector_v2.h" diff --git a/src/tbb/src/old/test_mutex_v2.cpp b/src/tbb/src/old/test_mutex_v2.cpp index e6d9c3d5..99902a34 100644 --- a/src/tbb/src/old/test_mutex_v2.cpp +++ b/src/tbb/src/old/test_mutex_v2.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define HARNESS_DEFAULT_MIN_THREADS 1 diff --git a/src/tbb/src/old/test_task_scheduler_observer_v3.cpp b/src/tbb/src/old/test_task_scheduler_observer_v3.cpp index cfd772be..547f16fd 100644 --- a/src/tbb/src/old/test_task_scheduler_observer_v3.cpp +++ b/src/tbb/src/old/test_task_scheduler_observer_v3.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,13 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -//TODO: when removing TBB_PREVIEW_LOCAL_OBSERVER, change the header or defines here +#define __TBB_ARENA_OBSERVER 0 #include "tbb/task_scheduler_observer.h" typedef uintptr_t FlagType; @@ -95,7 +91,7 @@ class DoTest { DoTest( int n ) : nthread(n) {} void operator()( int i ) const { LocalState->IsMaster = true; - if( i==0 ) { + if( i==0 ) { tbb::task_scheduler_init init(nthread); DoFib(0); } else { @@ -112,8 +108,8 @@ void TestObserver( int p, int q ) { } int TestMain () { - for( int p=MinThread; p<=MaxThread; ++p ) - for( int q=MinThread; q<=MaxThread; ++q ) + for( int p=MinThread; p<=MaxThread; ++p ) + for( int q=MinThread; q<=MaxThread; ++q ) TestObserver(p,q); ASSERT( EntryCount>0, "on_scheduler_entry not exercised" ); ASSERT( ExitCount>0, "on_scheduler_exit not exercised" ); diff --git a/src/tbb/src/perf/coarse_grained_raii_lru_cache.h b/src/tbb/src/perf/coarse_grained_raii_lru_cache.h index 9b11400e..556fc87c 100644 --- a/src/tbb/src/perf/coarse_grained_raii_lru_cache.h +++ b/src/tbb/src/perf/coarse_grained_raii_lru_cache.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef coarse_grained_raii_lru_cache_H diff --git a/src/tbb/src/perf/cpq_pdes.cpp b/src/tbb/src/perf/cpq_pdes.cpp index c54eb8fc..2ec842f2 100644 --- a/src/tbb/src/perf/cpq_pdes.cpp +++ b/src/tbb/src/perf/cpq_pdes.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/perf/fibonacci_impl_tbb.cpp b/src/tbb/src/perf/fibonacci_impl_tbb.cpp index 158ddb49..4ef99002 100644 --- a/src/tbb/src/perf/fibonacci_impl_tbb.cpp +++ b/src/tbb/src/perf/fibonacci_impl_tbb.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/perf/perf.cpp b/src/tbb/src/perf/perf.cpp index 36a3ba8e..16e7020a 100644 --- a/src/tbb/src/perf/perf.cpp +++ b/src/tbb/src/perf/perf.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "perf.h" diff --git a/src/tbb/src/perf/perf.h b/src/tbb/src/perf/perf.h index fb5f2ce6..7ea6d397 100644 --- a/src/tbb/src/perf/perf.h +++ b/src/tbb/src/perf/perf.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __tbb_perf_h__ diff --git a/src/tbb/src/perf/perf_sched.cpp b/src/tbb/src/perf/perf_sched.cpp index 321fed68..cbe8421e 100644 --- a/src/tbb/src/perf/perf_sched.cpp +++ b/src/tbb/src/perf/perf_sched.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "perf.h" diff --git a/src/tbb/src/perf/run_statistics.sh b/src/tbb/src/perf/run_statistics.sh index ac7ffd7c..a4152fc4 100644 --- a/src/tbb/src/perf/run_statistics.sh +++ b/src/tbb/src/perf/run_statistics.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2005-2017 Intel Corporation +# Copyright (c) 2005-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,15 +13,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# -# export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH #setting output format .csv, 'pivot' - is pivot table mode, ++ means append export STAT_FORMAT=pivot-csv++ -#check existing files because of apend mode +#check existing files because of append mode ls *.csv rm -i *.csv #setting a delimiter in txt or csv file diff --git a/src/tbb/src/perf/statistics.cpp b/src/tbb/src/perf/statistics.cpp index f0c190b4..564dab60 100644 --- a/src/tbb/src/perf/statistics.cpp +++ b/src/tbb/src/perf/statistics.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "statistics.h" diff --git a/src/tbb/src/perf/statistics.h b/src/tbb/src/perf/statistics.h index 7113a7ea..e384264e 100644 --- a/src/tbb/src/perf/statistics.h +++ b/src/tbb/src/perf/statistics.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Internal Intel tool diff --git a/src/tbb/src/perf/statistics_xml.h b/src/tbb/src/perf/statistics_xml.h index 396241de..7059596d 100644 --- a/src/tbb/src/perf/statistics_xml.h +++ b/src/tbb/src/perf/statistics_xml.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ const char XMLBRow[]= diff --git a/src/tbb/src/perf/time_async_return.cpp b/src/tbb/src/perf/time_async_return.cpp new file mode 100644 index 00000000..6ffe7ff7 --- /dev/null +++ b/src/tbb/src/perf/time_async_return.cpp @@ -0,0 +1,222 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + This microbenchmark measures prioritization of the tasks that were spawned to execute async_node + successor's body. The idea of the prioritization is to have TBB worker threads react eagerly on + the work, which was returned back to the graph through async_node's gateway interface while + being occupied by CPU work generated by function_node with nested parallelism. Worker threads + should prefer async_node task instead of next parallel_for task. The result of correct work + prioritization is the interleaving of CPU and ASYNC work: + + ASYNC task=1 started, thread_idx=0 + CPU task=1 started, thread_idx=0 + CPU task=1, nested pfor task=0, thread_idx=0 + CPU task=1, nested pfor task=4, thread_idx=1 + ASYNC task=1 finished, thread_idx=0 + ASYNC task=2 started, thread_idx=0 + CPU task=1, nested pfor task=1, thread_idx=0 + CPU task=1, nested pfor task=5, thread_idx=1 + ASYNC task=2 finished, thread_idx=0 + ASYNC task=3 started, thread_idx=0 + CPU task=1, nested pfor task=2, thread_idx=0 + CPU task=1, nested pfor task=6, thread_idx=1 + ASYNC task=3 finished, thread_idx=0 + ASYNC task=4 started, thread_idx=0 + CPU task=1, nested pfor task=3, thread_idx=0 + CPU task=1, nested pfor task=7, thread_idx=1 + ASYNC task=4 finished, thread_idx=0 + ASYNC task=5 started, thread_idx=0 + CPU task=1 finished, thread_idx=0 + CPU task=2 started, thread_idx=0 + CPU task=2, nested pfor task=0, thread_idx=0 + CPU task=2, nested pfor task=4, thread_idx=1 + CPU task=2, nested pfor task=5, thread_idx=1 + ASYNC task=5 finished, thread_idx=0 + ASYNC task=6 started, thread_idx=0 + CPU task=2, nested pfor task=1, thread_idx=0 + ASYNC task=6 finished, thread_idx=1 + CPU task=2, nested pfor task=2, thread_idx=0 + ASYNC task=7 started, thread_idx=1 + CPU task=2, nested pfor task=6, thread_idx=1 + ASYNC task=7 finished, thread_idx=0 + ASYNC task=8 started, thread_idx=0 + CPU task=2, nested pfor task=7, thread_idx=1 + CPU task=2, nested pfor task=3, thread_idx=0 + CPU task=2 finished, thread_idx=0 + ASYNC task=8 finished, thread_idx=1 + Elapsed time: 8.002 + + The parameters are chosen so that CPU and ASYNC work take approximately the same time. +*/ + +#define TBB_PREVIEW_FLOW_GRAPH_FEATURES __TBB_CPF_BUILD + +#include "tbb/task_scheduler_init.h" +#include "tbb/parallel_for.h" +#include "tbb/concurrent_queue.h" +#include "tbb/tick_count.h" +#include "tbb/tbb_thread.h" +#include "tbb/flow_graph.h" +#include "tbb/task_arena.h" +#include +#include + +const int NUM_THREADS = 2; // number of threads TBB is initialized with +const int CPU_LIMIT = 2; // number of repetitions of sub-graph with function_node +const double CPU_SPIN = 1.; // execution time of every parallel_for task +const int NESTED_CPU_TASKS_COUNT = 4 * NUM_THREADS; // number of parallel_for tasks +const int ASYNC_LIMIT = 8; // number of repetitions of sub-graph with async_node +const double ASYNC_SPIN = 0.5; // execution time of every async_node work + +void spin(double s) { + tbb::tick_count start = tbb::tick_count::now(); + while ((tbb::tick_count::now() - start).seconds() < s); +} + +typedef int data_type; +typedef tbb::flow::async_node async_node_type; +typedef tbb::flow::multifunction_node > decider_node_type; + +struct AsyncActivity { + typedef async_node_type::gateway_type gateway_type; + + struct work_type { + data_type input; + gateway_type* gateway; + }; + bool done; + bool end_of_work() { return done; } + tbb::concurrent_queue my_queue; + tbb::tbb_thread my_service_thread; + + struct ServiceThreadFunc { + void operator()(AsyncActivity* activity) { + while (!activity->end_of_work()) { + work_type work; + while (activity->my_queue.try_pop(work)) { + spin(ASYNC_SPIN); // do work + work.gateway->try_put(work.input); + work.gateway->release_wait(); + } + } + } + }; + + void stop_and_wait() { + done = true; + my_service_thread.join(); + } + + void submit(data_type input, gateway_type* gateway) { + work_type work = { input, gateway }; + gateway->reserve_wait(); + my_queue.push(work); + } + + AsyncActivity() : done(false), my_service_thread(ServiceThreadFunc(), this) {} +}; + +struct StartBody { + bool has_run; + bool operator()(data_type& input) { + if (has_run) return false; + else { + input = 1; + has_run = true; + return true; + } + } + StartBody() : has_run(false) {} +}; + +struct ParallelForBody { + const data_type& my_input; + ParallelForBody(const data_type& input) : my_input(input) {} + void operator()(const data_type& p) const { + std::printf(" CPU task=%d, nested pfor task=%d, thread_idx=%d\n", my_input, p, + tbb::this_task_arena::current_thread_index()); + spin(CPU_SPIN); + } +}; + +struct CpuWorkBody { + const int parallel_for_tasks_count; + data_type operator()(const data_type& input) { + std::printf("CPU task=%d started, thread_idx=%d\n", input, + tbb::this_task_arena::current_thread_index()); + tbb::parallel_for(0, parallel_for_tasks_count, ParallelForBody(input)); + return input; + } + CpuWorkBody() : parallel_for_tasks_count(NESTED_CPU_TASKS_COUNT) {} +}; + +struct DeciderBody { + const int& my_limit; + DeciderBody( const int& limit ) : my_limit( limit ) {} + void operator()(data_type input, decider_node_type::output_ports_type& ports) { + const char* work_type = my_limit == ASYNC_LIMIT ? "ASYNC" : "CPU"; + std::printf("%s task=%d finished, thread_idx=%d\n", work_type, input, + tbb::this_task_arena::current_thread_index()); + if (input < my_limit) + tbb::flow::get<0>(ports).try_put(input + 1); + else + tbb::flow::get<1>(ports).try_put(input + 1); + } +}; + +struct AsyncSubmissionBody { + AsyncActivity* my_activity; + void operator()(data_type input, async_node_type::gateway_type& gateway) { + my_activity->submit(input, &gateway); + std::printf("ASYNC task=%d started, thread_idx=%d\n", input, + tbb::this_task_arena::current_thread_index()); + } + AsyncSubmissionBody(AsyncActivity* activity) : my_activity(activity) {} +}; + +int main() { + tbb::task_scheduler_init init(NUM_THREADS); + AsyncActivity activity; + tbb::flow::graph g; + + tbb::flow::source_node starter_node(g, StartBody(), false); + tbb::flow::function_node cpu_work_node(g, tbb::flow::unlimited, CpuWorkBody()); + decider_node_type cpu_restarter_node(g, tbb::flow::unlimited, DeciderBody(CPU_LIMIT)); + async_node_type async_node(g, tbb::flow::unlimited, AsyncSubmissionBody(&activity)); + decider_node_type async_restarter_node(g, tbb::flow::unlimited, DeciderBody(ASYNC_LIMIT) +#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + , /*priority=*/1 +#endif + ); + + tbb::flow::make_edge(starter_node, cpu_work_node); + tbb::flow::make_edge(cpu_work_node, cpu_restarter_node); + tbb::flow::make_edge(tbb::flow::output_port<0>(cpu_restarter_node), cpu_work_node); + + tbb::flow::make_edge(starter_node, async_node); + tbb::flow::make_edge(async_node, async_restarter_node); + tbb::flow::make_edge(tbb::flow::output_port<0>(async_restarter_node), async_node); + + tbb::tick_count start_time = tbb::tick_count::now(); + starter_node.activate(); + g.wait_for_all(); + activity.stop_and_wait(); + std::printf("Elapsed time: %lf seconds\n", (tbb::tick_count::now() - start_time).seconds()); + + return 0; +} diff --git a/src/tbb/src/perf/time_cpq_throughput_test.cpp b/src/tbb/src/perf/time_cpq_throughput_test.cpp index 02f9bfd3..24376f36 100644 --- a/src/tbb/src/perf/time_cpq_throughput_test.cpp +++ b/src/tbb/src/perf/time_cpq_throughput_test.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define HARNESS_CUSTOM_MAIN 1 diff --git a/src/tbb/src/perf/time_fibonacci_cutoff.cpp b/src/tbb/src/perf/time_fibonacci_cutoff.cpp index 2a9e74d0..c5194280 100644 --- a/src/tbb/src/perf/time_fibonacci_cutoff.cpp +++ b/src/tbb/src/perf/time_fibonacci_cutoff.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/perf/time_framework.h b/src/tbb/src/perf/time_framework.h index a749dff6..ca715edb 100644 --- a/src/tbb/src/perf/time_framework.h +++ b/src/tbb/src/perf/time_framework.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TIME_FRAMEWORK_H__ diff --git a/src/tbb/src/perf/time_hash_map.cpp b/src/tbb/src/perf/time_hash_map.cpp index 690d092f..67041532 100644 --- a/src/tbb/src/perf/time_hash_map.cpp +++ b/src/tbb/src/perf/time_hash_map.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // configuration: diff --git a/src/tbb/src/perf/time_hash_map_fill.cpp b/src/tbb/src/perf/time_hash_map_fill.cpp index f0919375..93d9cc46 100644 --- a/src/tbb/src/perf/time_hash_map_fill.cpp +++ b/src/tbb/src/perf/time_hash_map_fill.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // configuration: diff --git a/src/tbb/src/perf/time_hash_map_fill.html b/src/tbb/src/perf/time_hash_map_fill.html index 766e85e1..f5b43e6b 100644 --- a/src/tbb/src/perf/time_hash_map_fill.html +++ b/src/tbb/src/perf/time_hash_map_fill.html @@ -8,7 +8,7 @@

time_hash_map_fill

Diagram

Prepare results

-

This benchmark outputs results in Excel* and html file formats by default. To generate text (CSV) file instead, specify STAT_FORMAT=pivot-csv evironment variable. To change the default table size, set TABLE_SIZE. +

This benchmark outputs results in Excel* and html file formats by default. To generate text (CSV) file instead, specify STAT_FORMAT=pivot-csv environment variable. To change the default table size, set TABLE_SIZE. 

src$    make time_hash_map_fill args=-v STAT_FORMAT=pivot-csv TABLE_SIZE=250000
Or to get statistics from different runs: 
src$    make time_hash_map_fill TABLE_SIZE=50000 run_cmd="bash ../../src/perf/run_statistics.sh"

@@ -112,7 +112,7 @@

Script body

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/perf/time_locked_work.cpp b/src/tbb/src/perf/time_locked_work.cpp index 3a966bbb..d09870ed 100644 --- a/src/tbb/src/perf/time_locked_work.cpp +++ b/src/tbb/src/perf/time_locked_work.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ ////// Test configuration //////////////////////////////////////////////////// diff --git a/src/tbb/src/perf/time_lru_cache_throughput.cpp b/src/tbb/src/perf/time_lru_cache_throughput.cpp index 151ef061..c202a11d 100644 --- a/src/tbb/src/perf/time_lru_cache_throughput.cpp +++ b/src/tbb/src/perf/time_lru_cache_throughput.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "../examples/common/utility/utility.h" @@ -38,7 +34,7 @@ #include #include "tbb/mutex.h" -//TODO: probably move this to separate header utlity file +//TODO: probably move this to separate header utility file namespace micro_benchmarking{ namespace utils{ template diff --git a/src/tbb/src/perf/time_parallel_for_each.cpp b/src/tbb/src/perf/time_parallel_for_each.cpp index 10e99c0d..55fb2d0d 100644 --- a/src/tbb/src/perf/time_parallel_for_each.cpp +++ b/src/tbb/src/perf/time_parallel_for_each.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/perf/time_sandbox.h b/src/tbb/src/perf/time_sandbox.h index 120b1b50..4a905ed2 100644 --- a/src/tbb/src/perf/time_sandbox.h +++ b/src/tbb/src/perf/time_sandbox.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TIME_FRAMEWORK_H__ diff --git a/src/tbb/src/perf/time_split_node.cpp b/src/tbb/src/perf/time_split_node.cpp index dab9ac42..79cc7773 100644 --- a/src/tbb/src/perf/time_split_node.cpp +++ b/src/tbb/src/perf/time_split_node.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/perf/time_vector.cpp b/src/tbb/src/perf/time_vector.cpp index 02ae84c7..c83e66ae 100644 --- a/src/tbb/src/perf/time_vector.cpp +++ b/src/tbb/src/perf/time_vector.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ //#define DO_SCALABLEALLOC diff --git a/src/tbb/src/rml/client/index.html b/src/tbb/src/rml/client/index.html index 53152ca9..8e7b50e6 100644 --- a/src/tbb/src/rml/client/index.html +++ b/src/tbb/src/rml/client/index.html @@ -31,7 +31,7 @@

Specific to client=TBB

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/rml/client/library_assert.h b/src/tbb/src/rml/client/library_assert.h index df3fcdbd..f93e53d5 100644 --- a/src/tbb/src/rml/client/library_assert.h +++ b/src/tbb/src/rml/client/library_assert.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef LIBRARY_ASSERT_H diff --git a/src/tbb/src/rml/client/omp_dynamic_link.cpp b/src/tbb/src/rml/client/omp_dynamic_link.cpp index e9a8d2e4..0716f8b5 100644 --- a/src/tbb/src/rml/client/omp_dynamic_link.cpp +++ b/src/tbb/src/rml/client/omp_dynamic_link.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "omp_dynamic_link.h" diff --git a/src/tbb/src/rml/client/omp_dynamic_link.h b/src/tbb/src/rml/client/omp_dynamic_link.h index 30202931..4ddb7f11 100644 --- a/src/tbb/src/rml/client/omp_dynamic_link.h +++ b/src/tbb/src/rml/client/omp_dynamic_link.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __KMP_omp_dynamic_link_H diff --git a/src/tbb/src/rml/client/rml_factory.h b/src/tbb/src/rml/client/rml_factory.h index 0c4626a9..79267b02 100644 --- a/src/tbb/src/rml/client/rml_factory.h +++ b/src/tbb/src/rml/client/rml_factory.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // No ifndef guard because this file is not a normal include file. @@ -33,7 +29,7 @@ #define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".dylib" #elif __linux__ #define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".so.1" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX #define RML_SERVER_NAME "libirml" DEBUG_SUFFIX ".so" #else #error Unknown OS diff --git a/src/tbb/src/rml/client/rml_omp.cpp b/src/tbb/src/rml/client/rml_omp.cpp index 86935ed2..b42a518d 100644 --- a/src/tbb/src/rml/client/rml_omp.cpp +++ b/src/tbb/src/rml/client/rml_omp.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "rml_omp.h" diff --git a/src/tbb/src/rml/client/rml_tbb.cpp b/src/tbb/src/rml/client/rml_tbb.cpp index 5842d1ed..064dd67f 100644 --- a/src/tbb/src/rml/client/rml_tbb.cpp +++ b/src/tbb/src/rml/client/rml_tbb.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "../include/rml_tbb.h" diff --git a/src/tbb/src/rml/include/index.html b/src/tbb/src/rml/include/index.html index 46c8b0a7..5e218414 100644 --- a/src/tbb/src/rml/include/index.html +++ b/src/tbb/src/rml/include/index.html @@ -18,7 +18,7 @@

Files

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/rml/include/rml_base.h b/src/tbb/src/rml/include/rml_base.h index 951026fa..1dfb7012 100644 --- a/src/tbb/src/rml/include/rml_base.h +++ b/src/tbb/src/rml/include/rml_base.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Header guard and namespace names follow rml conventions. diff --git a/src/tbb/src/rml/include/rml_omp.h b/src/tbb/src/rml/include/rml_omp.h index 103936bb..5a346c39 100644 --- a/src/tbb/src/rml/include/rml_omp.h +++ b/src/tbb/src/rml/include/rml_omp.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Header guard and namespace names follow OpenMP runtime conventions. diff --git a/src/tbb/src/rml/include/rml_tbb.h b/src/tbb/src/rml/include/rml_tbb.h index ce47f175..4cdb9250 100644 --- a/src/tbb/src/rml/include/rml_tbb.h +++ b/src/tbb/src/rml/include/rml_tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Header guard and namespace names follow TBB conventions. diff --git a/src/tbb/src/rml/index.html b/src/tbb/src/rml/index.html index 7c8e7ea2..49099bfc 100644 --- a/src/tbb/src/rml/index.html +++ b/src/tbb/src/rml/index.html @@ -20,7 +20,7 @@

Directories

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/rml/perfor/omp_nested.cpp b/src/tbb/src/rml/perfor/omp_nested.cpp index f0f675fe..d336f287 100644 --- a/src/tbb/src/rml/perfor/omp_nested.cpp +++ b/src/tbb/src/rml/perfor/omp_nested.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/perfor/omp_simple.cpp b/src/tbb/src/rml/perfor/omp_simple.cpp index 334b5717..512407bd 100644 --- a/src/tbb/src/rml/perfor/omp_simple.cpp +++ b/src/tbb/src/rml/perfor/omp_simple.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/perfor/tbb_multi_omp.cpp b/src/tbb/src/rml/perfor/tbb_multi_omp.cpp index b324d13f..35a056e9 100644 --- a/src/tbb/src/rml/perfor/tbb_multi_omp.cpp +++ b/src/tbb/src/rml/perfor/tbb_multi_omp.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/perfor/tbb_simple.cpp b/src/tbb/src/rml/perfor/tbb_simple.cpp index b00bad93..f18393d9 100644 --- a/src/tbb/src/rml/perfor/tbb_simple.cpp +++ b/src/tbb/src/rml/perfor/tbb_simple.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/perfor/thread_level.h b/src/tbb/src/rml/perfor/thread_level.h index 6a2912ee..fa059be5 100644 --- a/src/tbb/src/rml/perfor/thread_level.h +++ b/src/tbb/src/rml/perfor/thread_level.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Thread level recorder diff --git a/src/tbb/src/rml/server/index.html b/src/tbb/src/rml/server/index.html index dc248746..75edbe9c 100644 --- a/src/tbb/src/rml/server/index.html +++ b/src/tbb/src/rml/server/index.html @@ -7,7 +7,7 @@

Overview

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/rml/server/irml.rc b/src/tbb/src/rml/server/irml.rc index 73a84f83..e218957f 100644 --- a/src/tbb/src/rml/server/irml.rc +++ b/src/tbb/src/rml/server/irml.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // Microsoft Visual C++ generated resource script. // @@ -81,7 +77,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Intel(R) Threading Building Blocks resource manager library\0" VALUE "FileVersion", TBB_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalCopyright", "Copyright 2005-2019 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG VALUE "OriginalFilename", "irml.dll\0" diff --git a/src/tbb/src/rml/server/job_automaton.h b/src/tbb/src/rml/server/job_automaton.h index 8fe81c96..26e798a0 100644 --- a/src/tbb/src/rml/server/job_automaton.h +++ b/src/tbb/src/rml/server/job_automaton.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __RML_job_automaton_H diff --git a/src/tbb/src/rml/server/lin-rml-export.def b/src/tbb/src/rml/server/lin-rml-export.def index 5fb069b4..01c8753e 100644 --- a/src/tbb/src/rml/server/lin-rml-export.def +++ b/src/tbb/src/rml/server/lin-rml-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/rml/server/rml_server.cpp b/src/tbb/src/rml/server/rml_server.cpp index b2597416..5db496fe 100644 --- a/src/tbb/src/rml/server/rml_server.cpp +++ b/src/tbb/src/rml/server/rml_server.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "rml_tbb.h" @@ -1681,7 +1677,7 @@ void wakeup_some_tbb_threads() { /* First, atomically grab the connection, then increase the server ref count to keep it from being released prematurely. Second, check if the balance is available for TBB - and the tbb conneciton has slack to exploit. If the answer is true, go ahead and + and the tbb connection has slack to exploit. If the answer is true, go ahead and try to wake some up. */ if( generic_connection::get_addr(active_tbb_connections)==0 ) // the next connection will see the change; return. diff --git a/src/tbb/src/rml/server/thread_monitor.h b/src/tbb/src/rml/server/thread_monitor.h index e7ea79e4..2a9df54e 100644 --- a/src/tbb/src/rml/server/thread_monitor.h +++ b/src/tbb/src/rml/server/thread_monitor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // All platform-specific threading support is encapsulated here. */ @@ -28,7 +24,7 @@ #include #include //_alloca #include "tbb/tbb_misc.h" // support for processor groups -#if __TBB_WIN8UI_SUPPORT +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) #include #endif #elif USE_PTHREAD @@ -78,7 +74,7 @@ class thread_monitor { friend class thread_monitor; tbb::atomic my_epoch; }; - thread_monitor() : spurious(false), my_sema() { + thread_monitor() : skipped_wakeup(false), my_sema() { my_cookie.my_epoch = 0; ITT_SYNC_CREATE(&my_sema, SyncType_RML, SyncObj_ThreadMonitor); in_wait = false; @@ -129,9 +125,9 @@ class thread_monitor { //! Detach thread static void detach_thread(handle_type handle); private: - cookie my_cookie; - tbb::atomic in_wait; - bool spurious; + cookie my_cookie; // epoch counter + tbb::atomic in_wait; + bool skipped_wakeup; tbb::internal::binary_semaphore my_sema; #if USE_PTHREAD static void check( int error_code, const char* routine ); @@ -144,13 +140,14 @@ class thread_monitor { #define STACK_SIZE_PARAM_IS_A_RESERVATION 0x00010000 #endif -#if __TBB_WIN8UI_SUPPORT +// _beginthreadex API is not available in Windows 8 Store* applications, so use std::thread instead +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_function, void* arg, size_t, const size_t*) { //TODO: check that exception thrown from std::thread is not swallowed silently std::thread* thread_tmp=new std::thread(thread_function, arg); return thread_tmp->native_handle(); } -#else //__TBB_WIN8UI_SUPPORT +#else inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type thread_routine, void* arg, size_t stack_size, const size_t* worker_index ) { unsigned thread_id; int number_of_processor_groups = ( worker_index ) ? tbb::internal::NumberOfProcessorGroups() : 0; @@ -167,7 +164,7 @@ inline thread_monitor::handle_type thread_monitor::launch( thread_routine_type t } return h; } -#endif //__TBB_WIN8UI_SUPPORT +#endif //__TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) void thread_monitor::join(handle_type handle) { #if TBB_USE_ASSERT @@ -192,10 +189,10 @@ void thread_monitor::detach_thread(handle_type handle) { inline void thread_monitor::yield() { // TODO: consider unification via __TBB_Yield or tbb::this_tbb_thread::yield -#if !__TBB_WIN8UI_SUPPORT - SwitchToThread(); -#else +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) std::this_thread::yield(); +#else + SwitchToThread(); #endif } #endif /* USE_WINTHREAD */ @@ -244,24 +241,25 @@ inline void thread_monitor::notify() { } inline void thread_monitor::prepare_wait( cookie& c ) { - if( spurious ) { - spurious = false; - // consumes a spurious posted signal. don't wait on my_sema. - my_sema.P(); + if( skipped_wakeup ) { + // Lazily consume a signal that was skipped due to cancel_wait + skipped_wakeup = false; + my_sema.P(); // does not really wait on the semaphore } c = my_cookie; - in_wait = true; - __TBB_full_memory_fence(); + in_wait.store( true ); } inline void thread_monitor::commit_wait( cookie& c ) { - bool do_it = ( c.my_epoch == my_cookie.my_epoch); + bool do_it = ( c.my_epoch == my_cookie.my_epoch ); if( do_it ) my_sema.P(); else cancel_wait(); } inline void thread_monitor::cancel_wait() { - spurious = ! in_wait.fetch_and_store( false ); + // if not in_wait, then some thread has sent us a signal; + // it will be consumed by the next prepare_wait call + skipped_wakeup = ! in_wait.fetch_and_store( false ); } } // namespace internal diff --git a/src/tbb/src/rml/server/wait_counter.h b/src/tbb/src/rml/server/wait_counter.h index 3ddd8e03..b199ffdc 100644 --- a/src/tbb/src/rml/server/wait_counter.h +++ b/src/tbb/src/rml/server/wait_counter.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __RML_wait_counter_H diff --git a/src/tbb/src/rml/server/win32-rml-export.def b/src/tbb/src/rml/server/win32-rml-export.def index 2a550d0b..a52aa2ae 100644 --- a/src/tbb/src/rml/server/win32-rml-export.def +++ b/src/tbb/src/rml/server/win32-rml-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS diff --git a/src/tbb/src/rml/server/win64-rml-export.def b/src/tbb/src/rml/server/win64-rml-export.def index 2a550d0b..a52aa2ae 100644 --- a/src/tbb/src/rml/server/win64-rml-export.def +++ b/src/tbb/src/rml/server/win64-rml-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS diff --git a/src/tbb/src/rml/test/rml_omp_stub.cpp b/src/tbb/src/rml/test/rml_omp_stub.cpp index eb701b17..d4179067 100644 --- a/src/tbb/src/rml/test/rml_omp_stub.cpp +++ b/src/tbb/src/rml/test/rml_omp_stub.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // This file is compiled with C++, but linked with a program written in C. diff --git a/src/tbb/src/rml/test/test_job_automaton.cpp b/src/tbb/src/rml/test/test_job_automaton.cpp index f416e58b..4d849aa2 100644 --- a/src/tbb/src/rml/test/test_job_automaton.cpp +++ b/src/tbb/src/rml/test/test_job_automaton.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "harness.h" diff --git a/src/tbb/src/rml/test/test_rml_mixed.cpp b/src/tbb/src/rml/test/test_rml_mixed.cpp index 0575df50..43f8c8d2 100644 --- a/src/tbb/src/rml/test/test_rml_mixed.cpp +++ b/src/tbb/src/rml/test/test_rml_mixed.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/test/test_rml_omp.cpp b/src/tbb/src/rml/test/test_rml_omp.cpp index 89096de5..668f6b07 100644 --- a/src/tbb/src/rml/test/test_rml_omp.cpp +++ b/src/tbb/src/rml/test/test_rml_omp.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/test/test_rml_omp_c_linkage.c b/src/tbb/src/rml/test/test_rml_omp_c_linkage.c index a59a5864..dc3eca2c 100644 --- a/src/tbb/src/rml/test/test_rml_omp_c_linkage.c +++ b/src/tbb/src/rml/test/test_rml_omp_c_linkage.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ void Cplusplus(); diff --git a/src/tbb/src/rml/test/test_rml_tbb.cpp b/src/tbb/src/rml/test/test_rml_tbb.cpp index f679baeb..61af4ca2 100644 --- a/src/tbb/src/rml/test/test_rml_tbb.cpp +++ b/src/tbb/src/rml/test/test_rml_tbb.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/rml/test/test_server.h b/src/tbb/src/rml/test/test_server.h index a72142fa..7b01e6cc 100644 --- a/src/tbb/src/rml/test/test_server.h +++ b/src/tbb/src/rml/test/test_server.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,15 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* This header contains code shared by test_omp_server.cpp and test_tbb_server.cpp There is no ifndef guard - test is supposed to include this file exactly once. - The test is also exected to have #include of rml_omp.h or rml_tbb.h before + The test is also executed to have #include of rml_omp.h or rml_tbb.h before including this header. This header should not use any parts of TBB that require linking in the TBB run-time. @@ -93,7 +89,7 @@ class MyJob: public ::rml::job { } ~MyJob() { // Overwrite so that accidental use after destruction can be detected. - memset(this,-1,sizeof(*this)); + memset(static_cast(this),-1,sizeof(*this)); } }; diff --git a/src/tbb/src/rml/test/test_thread_monitor.cpp b/src/tbb/src/rml/test/test_thread_monitor.cpp index ca632593..f32f51df 100644 --- a/src/tbb/src/rml/test/test_thread_monitor.cpp +++ b/src/tbb/src/rml/test/test_thread_monitor.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "harness.h" diff --git a/src/tbb/src/tbb/arena.cpp b/src/tbb/src/tbb/arena.cpp index abe4d35b..28717e75 100644 --- a/src/tbb/src/tbb/arena.cpp +++ b/src/tbb/src/tbb/arena.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/global_control.h" // thread_stack_size @@ -218,12 +214,19 @@ arena::arena ( market& m, unsigned num_slots, unsigned num_reserved_slots ) { mailbox(i+1).construct(); ITT_SYNC_CREATE(&mailbox(i+1), SyncType_Scheduler, SyncObj_Mailbox); my_slots[i].hint_for_pop = i; +#if __TBB_PREVIEW_CRITICAL_TASKS + my_slots[i].hint_for_critical = i; +#endif #if __TBB_STATISTICS my_slots[i].my_counters = new ( NFS_Allocate(1, sizeof(statistics_counters), NULL) ) statistics_counters; #endif /* __TBB_STATISTICS */ } my_task_stream.initialize(my_num_slots); ITT_SYNC_CREATE(&my_task_stream, SyncType_Scheduler, SyncObj_TaskStream); +#if __TBB_PREVIEW_CRITICAL_TASKS + my_critical_task_stream.initialize(my_num_slots); + ITT_SYNC_CREATE(&my_critical_task_stream, SyncType_Scheduler, SyncObj_CriticalTaskStream); +#endif #if __TBB_ENQUEUE_ENFORCED_CONCURRENCY my_concurrency_mode = cm_normal; #endif @@ -268,6 +271,9 @@ void arena::free_arena () { drained += mailbox(i+1).drain(); } __TBB_ASSERT( my_task_stream.drain()==0, "Not all enqueued tasks were executed"); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT( my_critical_task_stream.drain()==0, "Not all critical tasks were executed"); +#endif #if __TBB_COUNT_TASK_NODES my_market->update_task_node_count( -drained ); #endif /* __TBB_COUNT_TASK_NODES */ @@ -431,6 +437,10 @@ bool arena::is_out_of_work() { } __TBB_ASSERT( k <= n, NULL ); bool work_absent = k == n; +#if __TBB_PREVIEW_CRITICAL_TASKS + bool no_critical_tasks = my_critical_task_stream.empty(0); + work_absent &= no_critical_tasks; +#endif #if __TBB_TASK_PRIORITY // Variable tasks_present indicates presence of tasks at any priority // level, while work_absent refers only to the current priority. @@ -560,17 +570,48 @@ void arena::enqueue_task( task& t, intptr_t prio, FastRandom &random ) } __TBB_ASSERT(t.prefix().affinity==affinity_id(0), "affinity is ignored for enqueued tasks"); #endif /* TBB_USE_ASSERT */ +#if __TBB_PREVIEW_CRITICAL_TASKS + if( prio == internal::priority_critical || internal::is_critical( t ) ) { + // TODO: consider using of 'scheduler::handled_as_critical' + internal::make_critical( t ); + generic_scheduler* s = governor::local_scheduler_if_initialized(); + ITT_NOTIFY(sync_releasing, &my_critical_task_stream); + if( s && s->my_arena_slot ) { + // Scheduler is initialized and it is attached to the arena, + // propagate isolation level to critical task +#if __TBB_TASK_ISOLATION + t.prefix().isolation = s->my_innermost_running_task->prefix().isolation; +#endif + unsigned& lane = s->my_arena_slot->hint_for_critical; + my_critical_task_stream.push( &t, 0, tbb::internal::subsequent_lane_selector(lane) ); + } else { + // Either scheduler is not initialized or it is not attached to the arena + // use random lane for the task + my_critical_task_stream.push( &t, 0, internal::random_lane_selector(random) ); + } + advertise_new_work(); + return; + } +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ ITT_NOTIFY(sync_releasing, &my_task_stream); #if __TBB_TASK_PRIORITY intptr_t p = prio ? normalize_priority(priority_t(prio)) : normalized_normal_priority; assert_priority_valid(p); +#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD + my_task_stream.push( &t, p, internal::random_lane_selector(random) ); +#else my_task_stream.push( &t, p, random ); +#endif if ( p != my_top_priority ) my_market->update_arena_priority( *this, p ); #else /* !__TBB_TASK_PRIORITY */ __TBB_ASSERT_EX(prio == 0, "the library is not configured to respect the task priority"); +#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD + my_task_stream.push( &t, 0, internal::random_lane_selector(random) ); +#else my_task_stream.push( &t, 0, random ); +#endif #endif /* !__TBB_TASK_PRIORITY */ advertise_new_work(); #if __TBB_TASK_PRIORITY @@ -602,7 +643,7 @@ class nested_arena_context : no_copy { my_scheduler.my_innermost_running_task = my_orig_state.my_innermost_running_task; } else { my_scheduler.nested_arena_exit(); - static_cast(my_scheduler) = my_orig_state; // restore arena settings + static_cast(my_scheduler) = my_orig_state; // restore arena settings #if __TBB_TASK_PRIORITY my_scheduler.my_local_reload_epoch = *my_orig_state.my_ref_reload_epoch; #endif @@ -620,6 +661,9 @@ class nested_arena_context : no_copy { my_scheduler.my_properties.outermost = true; my_scheduler.my_properties.type = type; my_scheduler.my_innermost_running_task = my_scheduler.my_dummy_task; +#if __TBB_PREVIEW_CRITICAL_TASKS + my_scheduler.my_properties.has_taken_critical_task = false; +#endif #if __TBB_TASK_GROUP_CONTEXT // Save dummy's context and replace it by arena's context my_orig_ctx = my_scheduler.my_dummy_task->prefix().context; @@ -641,28 +685,27 @@ void generic_scheduler::nested_arena_entry(arena* a, size_t slot_index) { attach_arena( a, slot_index, /*is_master*/true ); __TBB_ASSERT( my_arena == a, NULL ); governor::assume_scheduler( this ); -#if __TBB_ARENA_OBSERVER - my_last_local_observer = 0; // TODO: try optimize number of calls - my_arena->my_observers.notify_entry_observers( my_last_local_observer, /*worker=*/false ); -#endif // TODO? ITT_NOTIFY(sync_acquired, a->my_slots + index); // TODO: it requires market to have P workers (not P-1) // TODO: a preempted worker should be excluded from assignment to other arenas e.g. my_slack-- if( !is_worker() && slot_index >= my_arena->my_num_reserved_slots ) my_arena->my_market->adjust_demand(*my_arena, -1); +#if __TBB_ARENA_OBSERVER + my_last_local_observer = 0; // TODO: try optimize number of calls + my_arena->my_observers.notify_entry_observers( my_last_local_observer, /*worker=*/false ); +#endif } void generic_scheduler::nested_arena_exit() { - if( !is_worker() && my_arena_index >= my_arena->my_num_reserved_slots ) - my_arena->my_market->adjust_demand(*my_arena, 1); #if __TBB_ARENA_OBSERVER my_arena->my_observers.notify_exit_observers( my_last_local_observer, /*worker=*/false ); #endif /* __TBB_ARENA_OBSERVER */ - #if __TBB_TASK_PRIORITY if ( my_offloaded_tasks ) my_arena->orphan_offloaded_tasks( *this ); #endif + if( !is_worker() && my_arena_index >= my_arena->my_num_reserved_slots ) + my_arena->my_market->adjust_demand(*my_arena, 1); // Free the master slot. __TBB_ASSERT(my_arena->my_slots[my_arena_index].my_scheduler, "A slot is already empty"); __TBB_store_with_release(my_arena->my_slots[my_arena_index].my_scheduler, (generic_scheduler*)NULL); @@ -753,11 +796,11 @@ void task_arena_base::internal_attach( ) { void task_arena_base::internal_enqueue( task& t, intptr_t prio ) const { __TBB_ASSERT(my_arena, NULL); - generic_scheduler* s = governor::local_scheduler_if_initialized(); + generic_scheduler* s = governor::local_scheduler_weak(); // scheduler is only needed for FastRandom instance __TBB_ASSERT(s, "Scheduler is not initialized"); // we allocated a task so can expect the scheduler #if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(my_arena->my_default_ctx == t.prefix().context, NULL); - __TBB_ASSERT(!my_arena->my_default_ctx->is_group_execution_cancelled(), // TODO: any better idea? + // Is there a better place for checking the state of my_default_ctx? + __TBB_ASSERT(!(my_arena->my_default_ctx == t.prefix().context && my_arena->my_default_ctx->is_group_execution_cancelled()), "The task will not be executed because default task_group_context of task_arena is cancelled. Has previously enqueued task thrown an exception?"); #endif my_arena->enqueue_task( t, prio, s->my_random ); @@ -776,7 +819,7 @@ class delegated_task : public task { task * orig_dummy; task_group_context * orig_ctx; scheduler_properties orig_props; - outermost_context(delegated_task *_t, generic_scheduler &_s) + outermost_context(delegated_task *_t, generic_scheduler &_s) : t(_t), s(_s), orig_dummy(s.my_dummy_task), orig_props(s.my_properties) { __TBB_ASSERT(s.my_innermost_running_task == t, NULL); #if __TBB_TASK_GROUP_CONTEXT @@ -835,12 +878,12 @@ void task_arena_base::internal_execute(internal::delegate_base& d) const { dynamic_cast< internal::delegated_function< graph_funct, void>* >(&d); if (deleg_funct) { - internal_enqueue(*new(task::allocate_root(*my_context)) + internal_enqueue(*new(task::allocate_root(*my_context)) internal::function_task< internal::strip< graph_funct >::type > (internal::forward< graph_funct >(deleg_funct->my_func)), 0); return; } else { -#endif +#endif /* __TBB_USE_OPTIONAL_RTTI */ concurrent_monitor::thread_context waiter; #if __TBB_TASK_GROUP_CONTEXT task_group_context exec_context(task_group_context::isolated, my_version_and_traits & exact_exception_flag); @@ -890,8 +933,8 @@ void task_arena_base::internal_execute(internal::delegate_base& d) const { } // if (index1 == arena::out_of_arena) } // if (!same_arena) - cpu_ctl_env_helper cpu_ctl_helper; - cpu_ctl_helper.set_env(__TBB_CONTEXT_ARG1(my_context)); + context_guard_helper context_guard; + context_guard.set_ctx(__TBB_CONTEXT_ARG1(my_context)); #if TBB_USE_EXCEPTIONS try { #endif @@ -901,7 +944,7 @@ void task_arena_base::internal_execute(internal::delegate_base& d) const { #if TBB_USE_EXCEPTIONS } catch (...) { - cpu_ctl_helper.restore_default(); // TODO: is it needed on Windows? + context_guard.restore_default(); // TODO: is it needed on Windows? if (my_version_and_traits & exact_exception_flag) throw; else { task_group_context exception_container(task_group_context::isolated, @@ -982,7 +1025,7 @@ class isolation_guard : tbb::internal::no_copy { }; void isolate_within_arena( delegate_base& d, intptr_t reserved ) { - __TBB_ASSERT( reserved == 0, NULL ); + __TBB_ASSERT_EX( reserved == 0, NULL ); // TODO: Decide what to do if the scheduler is not initialized. Is there a use case for it? generic_scheduler* s = governor::local_scheduler_weak(); __TBB_ASSERT( s, "this_task_arena::isolate() needs an initialized scheduler" ); diff --git a/src/tbb/src/tbb/arena.h b/src/tbb/src/tbb/arena.h index 2bc23c4b..a5e9c061 100644 --- a/src/tbb/src/tbb/arena.h +++ b/src/tbb/src/tbb/arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_arena_H @@ -28,7 +24,11 @@ #include "scheduler_common.h" #include "intrusive_list.h" +#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD +#include "task_stream_extended.h" +#else #include "task_stream.h" +#endif #include "../rml/include/rml_tbb.h" #include "mailbox.h" #include "observer_proxy.h" @@ -70,7 +70,20 @@ struct arena_base : padded { relaxed FIFO order); - the enqueuing thread does not call any of wait_for_all methods. Depending on __TBB_TASK_PRIORITY, num_priority_levels can be 1 or more. **/ +#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD + task_stream my_task_stream; // heavy use in stealing loop +#else task_stream my_task_stream; // heavy use in stealing loop +#endif + +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Task pool for the tasks with critical property set. + /** Critical tasks are scheduled for execution ahead of other sources (including local task pool + and even bypassed tasks) unless the thread already executes a critical task in an outer + dispatch loop **/ + // used on the hot path of the task dispatch loop + task_stream<1, back_nonnull_accessor> my_critical_task_stream; +#endif //! The number of workers requested by the master thread owning the arena. unsigned my_max_num_workers; @@ -448,6 +461,7 @@ template void arena::advertise_new_work() { } } #endif /* __TBB_ENQUEUE_ENFORCED_CONCURRENCY */ + // TODO: investigate adjusting of arena's demand by a single worker. my_market->adjust_demand( *this, my_max_num_workers ); } } diff --git a/src/tbb/src/tbb/cache_aligned_allocator.cpp b/src/tbb/src/tbb/cache_aligned_allocator.cpp index e1e87132..e6cd4848 100644 --- a/src/tbb/src/tbb/cache_aligned_allocator.cpp +++ b/src/tbb/src/tbb/cache_aligned_allocator.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -32,8 +28,6 @@ #include #endif /* _WIN32||_WIN64 */ -using namespace std; - #if __TBB_WEAK_SYMBOLS_PRESENT #pragma weak scalable_malloc @@ -104,7 +98,7 @@ static const dynamic_link_descriptor MallocLinkTable[] = { #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" #elif __APPLE__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX || __ANDROID__ +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" #elif __linux__ // Note that order of these #elif's is important! #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) @@ -124,8 +118,8 @@ void initialize_handler_pointers() { // This must be done now, and not before FillDynamicLinks runs, because if other // threads call the handlers, we want them to go through the DoOneTimeInitializations logic, // which forces them to wait. - FreeHandler = &free; - MallocHandler = &malloc; + FreeHandler = &std::free; + MallocHandler = &std::malloc; padded_allocate_handler = &padded_allocate; padded_free_handler = &padded_free; } @@ -207,7 +201,7 @@ void NFS_Free( void* p ) { static void* padded_allocate( size_t bytes, size_t alignment ) { unsigned char* result = NULL; - unsigned char* base = (unsigned char*)malloc(alignment+bytes); + unsigned char* base = (unsigned char*)std::malloc(alignment+bytes); if( base ) { // Round up to the next line result = (unsigned char*)((uintptr_t)(base+alignment)&-alignment); @@ -223,7 +217,7 @@ static void padded_free( void* p ) { // Recover where block actually starts unsigned char* base = ((unsigned char**)p)[-1]; __TBB_ASSERT( (void*)((uintptr_t)(base+NFS_LineSize)&-NFS_LineSize)==p, "not allocated by NFS_Allocate?" ); - free(base); + std::free(base); } } @@ -248,9 +242,9 @@ bool __TBB_EXPORTED_FUNC is_malloc_used_v3() { } __TBB_ASSERT( MallocHandler!=&DummyMalloc && FreeHandler!=&DummyFree, NULL ); // Cast to void avoids type mismatch errors on some compilers (e.g. __IBMCPP__) - __TBB_ASSERT( !(((void*)MallocHandler==(void*)&malloc) ^ ((void*)FreeHandler==(void*)&free)), + __TBB_ASSERT( !(((void*)MallocHandler==(void*)&std::malloc) ^ ((void*)FreeHandler==(void*)&std::free)), "Both shim pointers must refer to routines from the same package (either TBB or CRT)" ); - return (void*)MallocHandler == (void*)&malloc; + return (void*)MallocHandler == (void*)&std::malloc; } } // namespace internal diff --git a/src/tbb/src/tbb/cilk-tbb-interop.h b/src/tbb/src/tbb/cilk-tbb-interop.h index 19db6ea9..295734b6 100644 --- a/src/tbb/src/tbb/cilk-tbb-interop.h +++ b/src/tbb/src/tbb/cilk-tbb-interop.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /* The API to enable interoperability between Intel(R) Cilk(TM) Plus and diff --git a/src/tbb/src/tbb/concurrent_hash_map.cpp b/src/tbb/src/tbb/concurrent_hash_map.cpp index d2dcb23d..5ad5884f 100644 --- a/src/tbb/src/tbb/concurrent_hash_map.cpp +++ b/src/tbb/src/tbb/concurrent_hash_map.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/concurrent_hash_map.h" diff --git a/src/tbb/src/tbb/concurrent_monitor.cpp b/src/tbb/src/tbb/concurrent_monitor.cpp index 5ae3784c..d9c85669 100644 --- a/src/tbb/src/tbb/concurrent_monitor.cpp +++ b/src/tbb/src/tbb/concurrent_monitor.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "concurrent_monitor.h" @@ -36,9 +32,9 @@ concurrent_monitor::~concurrent_monitor() { void concurrent_monitor::prepare_wait( thread_context& thr, uintptr_t ctx ) { if( !thr.ready ) thr.init(); - // this is good place to pump previous spurious wakeup - else if( thr.spurious ) { - thr.spurious = false; + // this is good place to pump previous skipped wakeup + else if( thr.skipped_wakeup ) { + thr.skipped_wakeup = false; thr.semaphore().P(); } thr.context = ctx; @@ -52,18 +48,17 @@ void concurrent_monitor::prepare_wait( thread_context& thr, uintptr_t ctx ) { } void concurrent_monitor::cancel_wait( thread_context& thr ) { - // spurious wakeup will be pumped in the following prepare_wait() - thr.spurious = true; + // possible skipped wakeup will be pumped in the following prepare_wait() + thr.skipped_wakeup = true; // try to remove node from waitset bool th_in_waitset = thr.in_waitset; if( th_in_waitset ) { tbb::spin_mutex::scoped_lock l( mutex_ec ); if (thr.in_waitset) { - // successfully removed from waitset, - // so there will be no spurious wakeup - thr.in_waitset = false; - thr.spurious = false; waitset_ec.remove( (waitset_t::node_t&)thr ); + // node is removed from waitset, so there will be no wakeup + thr.in_waitset = false; + thr.skipped_wakeup = false; } } } diff --git a/src/tbb/src/tbb/concurrent_monitor.h b/src/tbb/src/tbb/concurrent_monitor.h index 55c4dc92..712cd5b1 100644 --- a/src/tbb/src/tbb/concurrent_monitor.h +++ b/src/tbb/src/tbb/concurrent_monitor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_concurrent_monitor_H @@ -100,13 +96,13 @@ class concurrent_monitor : no_copy { class thread_context : waitset_node_t, no_copy { friend class concurrent_monitor; public: - thread_context() : spurious(false), aborted(false), ready(false), context(0) { + thread_context() : skipped_wakeup(false), aborted(false), ready(false), context(0) { epoch = 0; in_waitset = false; } ~thread_context() { if (ready) { - if( spurious ) semaphore().P(); + if( skipped_wakeup ) semaphore().P(); semaphore().~binary_semaphore(); } } @@ -119,7 +115,7 @@ class concurrent_monitor : no_copy { tbb::aligned_space sema; __TBB_atomic unsigned epoch; tbb::atomic in_waitset; - bool spurious; + bool skipped_wakeup; bool aborted; bool ready; uintptr_t context; diff --git a/src/tbb/src/tbb/concurrent_queue.cpp b/src/tbb/src/tbb/concurrent_queue.cpp index 2047bacb..1c1d8781 100644 --- a/src/tbb/src/tbb/concurrent_queue.cpp +++ b/src/tbb/src/tbb/concurrent_queue.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_stddef.h" @@ -29,8 +25,6 @@ #include #include // for memset() -using namespace std; - #if defined(_MSC_VER) && defined(_Wp64) // Workaround for overzealous compiler warnings in /Wp64 mode #pragma warning (disable: 4267) @@ -353,7 +347,7 @@ concurrent_queue_base_v3::concurrent_queue_base_v3( size_t item_sz ) { __TBB_ASSERT( is_aligned(&my_rep->head_counter, NFS_GetLineSize()), "alignment error" ); __TBB_ASSERT( is_aligned(&my_rep->tail_counter, NFS_GetLineSize()), "alignment error" ); __TBB_ASSERT( is_aligned(&my_rep->array, NFS_GetLineSize()), "alignment error" ); - memset((void*) my_rep, 0, sizeof(concurrent_queue_rep)); + std::memset(static_cast(my_rep),0,sizeof(concurrent_queue_rep)); new ( &my_rep->items_avail ) concurrent_monitor(); new ( &my_rep->slots_avail ) concurrent_monitor(); this->item_size = item_sz; diff --git a/src/tbb/src/tbb/concurrent_vector.cpp b/src/tbb/src/tbb/concurrent_vector.cpp index 4ecf0990..3dec58df 100644 --- a/src/tbb/src/tbb/concurrent_vector.cpp +++ b/src/tbb/src/tbb/concurrent_vector.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if (_MSC_VER) @@ -38,12 +34,10 @@ #pragma warning (disable: 4267) #endif -using namespace std; - namespace tbb { namespace internal { - class concurrent_vector_base_v3::helper :no_assign { +class concurrent_vector_base_v3::helper :no_assign { public: //! memory page size static const size_type page_size = 4096; @@ -197,7 +191,7 @@ namespace internal { func( begin, n ); } }; -}; +}; // class helper void concurrent_vector_base_v3::helper::extend_segment_table(concurrent_vector_base_v3 &v, concurrent_vector_base_v3::size_type start) { if( start > segment_size(pointers_per_short_table) ) start = segment_size(pointers_per_short_table); @@ -337,11 +331,9 @@ concurrent_vector_base_v3::size_type concurrent_vector_base_v3::internal_capacit } void concurrent_vector_base_v3::internal_throw_exception(size_type t) const { - switch(t) { - case 0: throw_exception(eid_out_of_range); - case 1: throw_exception(eid_segment_range_error); - case 2: throw_exception(eid_index_range_error); - } + exception_id ids[] = { eid_out_of_range, eid_segment_range_error, eid_index_range_error }; + __TBB_ASSERT(t < sizeof(ids) / sizeof(exception_id), NULL); + throw_exception(ids[t]); } void concurrent_vector_base_v3::internal_reserve( size_type n, size_type element_size, size_type max_size ) { diff --git a/src/tbb/src/tbb/condition_variable.cpp b/src/tbb/src/tbb/condition_variable.cpp index 49ed609f..36ea3efb 100644 --- a/src/tbb/src/tbb/condition_variable.cpp +++ b/src/tbb/src/tbb/condition_variable.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -141,8 +137,18 @@ static const dynamic_link_descriptor CondVarLinkTable[] = { void init_condvar_module() { __TBB_ASSERT( (uintptr_t)__TBB_init_condvar==(uintptr_t)&init_condvar_using_event, NULL ); - if( dynamic_link( "Kernel32.dll", CondVarLinkTable, 4 ) ) +#if __TBB_WIN8UI_SUPPORT + // We expect condition variables to be always available for Windows* store applications, + // so there is no need to check presence and use alternative implementation. + __TBB_init_condvar = (void (WINAPI *)(PCONDITION_VARIABLE))&InitializeConditionVariable; + __TBB_condvar_wait = (BOOL(WINAPI *)(PCONDITION_VARIABLE, LPCRITICAL_SECTION, DWORD))&SleepConditionVariableCS; + __TBB_condvar_notify_one = (void (WINAPI *)(PCONDITION_VARIABLE))&WakeConditionVariable; + __TBB_condvar_notify_all = (void (WINAPI *)(PCONDITION_VARIABLE))&WakeAllConditionVariable; + __TBB_destroy_condvar = (void (WINAPI *)(PCONDITION_VARIABLE))&destroy_condvar_noop; +#else + if (dynamic_link("Kernel32.dll", CondVarLinkTable, 4)) __TBB_destroy_condvar = (void (WINAPI *)(PCONDITION_VARIABLE))&destroy_condvar_noop; +#endif } #endif /* _WIN32||_WIN64 */ diff --git a/src/tbb/src/tbb/critical_section.cpp b/src/tbb/src/tbb/critical_section.cpp index 4b32fb17..f2cce998 100644 --- a/src/tbb/src/tbb/critical_section.cpp +++ b/src/tbb/src/tbb/critical_section.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/critical_section.h" diff --git a/src/tbb/src/tbb/custom_scheduler.h b/src/tbb/src/tbb/custom_scheduler.h index c6511b27..d703b9e4 100644 --- a/src/tbb/src/tbb/custom_scheduler.h +++ b/src/tbb/src/tbb/custom_scheduler.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_custom_scheduler_H @@ -109,6 +105,12 @@ class custom_scheduler: private generic_scheduler { #endif /*__TBB_RECYCLE_TO_ENQUEUE*/ if( bypass_slot==NULL ) bypass_slot = &s; +#if __TBB_PREVIEW_CRITICAL_TASKS + else if( internal::is_critical( s ) ) { + local_spawn( bypass_slot, bypass_slot->prefix().next ); + bypass_slot = &s; + } +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ else local_spawn( &s, s.prefix().next ); } @@ -216,9 +218,15 @@ task* custom_scheduler::receive_or_steal_task( __TBB_ISOLATION_ GATHER_STATISTIC( ++my_counters.mails_received ); } // Check if there are tasks in starvation-resistant stream. - // Only allowed at the outermost dispatch level. - else if ( outermost_dispatch_level && !my_arena->my_task_stream.empty(p) - && (t = my_arena->my_task_stream.pop( p, my_arena_slot->hint_for_pop)) ) { + // Only allowed at the outermost dispatch level without isolation. + else if (__TBB_ISOLATION_EXPR(isolation == no_isolation &&) outermost_dispatch_level && + !my_arena->my_task_stream.empty(p) && ( +#if __TBB_PREVIEW_CRITICAL_TASKS && __TBB_CPF_BUILD + t = my_arena->my_task_stream.pop( p, subsequent_lane_selector(my_arena_slot->hint_for_pop) ) +#else + t = my_arena->my_task_stream.pop( p, my_arena_slot->hint_for_pop ) +#endif + ) ) { ITT_NOTIFY(sync_acquired, &my_arena->my_task_stream); // just proceed with the obtained task } @@ -229,38 +237,16 @@ task* custom_scheduler::receive_or_steal_task( __TBB_ISOLATION_ // just proceed with the obtained task } #endif /* __TBB_TASK_PRIORITY */ - else if ( can_steal_here && n ) { - // Try to steal a task from a random victim. - size_t k = my_random.get() % n; - arena_slot* victim = &my_arena->my_slots[k]; - // The following condition excludes the master that might have - // already taken our previous place in the arena from the list . - // of potential victims. But since such a situation can take - // place only in case of significant oversubscription, keeping - // the checks simple seems to be preferable to complicating the code. - if( k >= my_arena_index ) - ++victim; // Adjusts random distribution to exclude self - task **pool = victim->task_pool; - if( pool == EmptyTaskPool || !(t = steal_task( __TBB_ISOLATION_ARG(*victim, isolation) )) ) - goto fail; - if( is_proxy(*t) ) { - task_proxy &tp = *(task_proxy*)t; - t = tp.extract_task(); - if ( !t ) { - // Proxy was empty, so it's our responsibility to free it - free_task(tp); - goto fail; - } - GATHER_STATISTIC( ++my_counters.proxies_stolen ); - } - t->prefix().extra_state |= es_task_is_stolen; - if( is_version_3_task(*t) ) { - my_innermost_running_task = t; - t->prefix().owner = this; - t->note_affinity( my_affinity_id ); - } - GATHER_STATISTIC( ++my_counters.steals_committed ); - } // end of stealing branch + else if ( can_steal_here && n && (t = steal_task( __TBB_ISOLATION_EXPR(isolation) )) ) { + // just proceed with the obtained task + } +#if __TBB_PREVIEW_CRITICAL_TASKS + else if( (t = get_critical_task( __TBB_ISOLATION_EXPR(isolation) )) ) { + __TBB_ASSERT( internal::is_critical(*t), "Received task must be critical one" ); + ITT_NOTIFY(sync_acquired, &my_arena->my_critical_task_stream); + // just proceed with the obtained task + } +#endif // __TBB_PREVIEW_CRITICAL_TASKS else goto fail; // A task was successfully obtained somewhere @@ -422,9 +408,9 @@ void custom_scheduler::local_wait_for_all( task& parent, task* #endif /* __TBB_TASK_PRIORITY */ } - cpu_ctl_env_helper cpu_ctl_helper; + context_guard_helper context_guard; if ( t ) { - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); + context_guard.set_ctx( __TBB_CONTEXT_ARG1(t->prefix().context) ); #if __TBB_TASK_ISOLATION if ( isolation != no_isolation ) { __TBB_ASSERT( t->prefix().isolation == no_isolation, NULL ); @@ -433,7 +419,6 @@ void custom_scheduler::local_wait_for_all( task& parent, task* } #endif /* __TBB_TASK_ISOLATION */ } - #if TBB_USE_EXCEPTIONS // Infinite safeguard EH loop for (;;) { @@ -463,33 +448,53 @@ void custom_scheduler::local_wait_for_all( task& parent, task* // TODO: make the assert stronger by prohibiting allocated state. __TBB_ASSERT( 1L<state() & (1L<my_critical_task_stream); + t->prefix().state = task::allocated; + my_innermost_running_task = t; // required during spawn to propagate isolation + local_spawn(t, t->prefix().next); + t = critical_task; + } else { +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ #if __TBB_TASK_PRIORITY - intptr_t p = priority(*t); - if ( p != *my_ref_top_priority && (t->prefix().extra_state & es_task_enqueued) == 0) { - assert_priority_valid(p); - if ( p != my_arena->my_top_priority ) { - my_market->update_arena_priority( *my_arena, p ); - } - if ( p < effective_reference_priority() ) { - if ( !my_offloaded_tasks ) { - my_offloaded_task_list_tail_link = &t->prefix().next_offloaded; - // Erase possible reference to the owner scheduler (next_offloaded is a union member) - *my_offloaded_task_list_tail_link = NULL; - } - offload_task( *t, p ); - if ( is_task_pool_published() ) { - t = winnow_task_pool( __TBB_ISOLATION_EXPR( isolation ) ); - if ( t ) - continue; - } else { - // Mark arena as full to unlock arena priority level adjustment - // by arena::is_out_of_work(), and ensure worker's presence. - my_arena->advertise_new_work(); + intptr_t p = priority(*t); + if ( p != *my_ref_top_priority + && (t->prefix().extra_state & es_task_enqueued) == 0 ) { + assert_priority_valid(p); + if ( p != my_arena->my_top_priority ) { + my_market->update_arena_priority( *my_arena, p ); + } + if ( p < effective_reference_priority() ) { + if ( !my_offloaded_tasks ) { + my_offloaded_task_list_tail_link = &t->prefix().next_offloaded; + // Erase possible reference to the owner scheduler + // (next_offloaded is a union member) + *my_offloaded_task_list_tail_link = NULL; + } + offload_task( *t, p ); + if ( is_task_pool_published() ) { + t = winnow_task_pool( __TBB_ISOLATION_EXPR( isolation ) ); + if ( t ) + continue; + } else { + // Mark arena as full to unlock arena priority level adjustment + // by arena::is_out_of_work(), and ensure worker's presence. + my_arena->advertise_new_work(); + } + goto stealing_ground; + } } - goto stealing_ground; - } - } #endif /* __TBB_TASK_PRIORITY */ +#if __TBB_PREVIEW_CRITICAL_TASKS + } + } // if is not critical +#endif task* t_next = NULL; my_innermost_running_task = t; t->prefix().owner = this; @@ -506,11 +511,14 @@ void custom_scheduler::local_wait_for_all( task& parent, task* GATHER_STATISTIC( my_counters.avg_market_prio += my_market->my_global_top_priority ); #endif /* __TBB_TASK_PRIORITY */ ITT_STACK(SchedulerTraits::itt_possible, callee_enter, t->prefix().context->itt_caller); +#if __TBB_PREVIEW_CRITICAL_TASKS + internal::critical_task_count_guard tc_guard(my_properties, *t); +#endif t_next = t->execute(); ITT_STACK(SchedulerTraits::itt_possible, callee_leave, t->prefix().context->itt_caller); if (t_next) { __TBB_ASSERT( t_next->state()==task::allocated, - "if task::execute() returns task, it must be marked as allocated" ); + "if task::execute() returns task, it must be marked as allocated" ); reset_extra_state(t_next); __TBB_ISOLATION_EXPR( t_next->prefix().isolation = t->prefix().isolation ); #if TBB_USE_ASSERT @@ -518,7 +526,7 @@ void custom_scheduler::local_wait_for_all( task& parent, task* if (next_affinity != 0 && next_affinity != my_affinity_id) GATHER_STATISTIC( ++my_counters.affinity_ignored ); #endif - } + } // if there is bypassed task } assert_task_pool_valid(); switch( t->state() ) { @@ -538,6 +546,7 @@ void custom_scheduler::local_wait_for_all( task& parent, task* case task::recycle: // set by recycle_as_safe_continuation() t->prefix().state = task::allocated; #if __TBB_RECYCLE_TO_ENQUEUE + __TBB_fallthrough; case task::to_enqueue: // set by recycle_to_enqueue() #endif __TBB_ASSERT( t_next != t, "a task returned from method execute() can not be recycled in another way" ); @@ -590,7 +599,7 @@ void custom_scheduler::local_wait_for_all( task& parent, task* if ( !t ) break; - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); + context_guard.set_ctx( __TBB_CONTEXT_ARG1(t->prefix().context) ); }; // end of local task pool retrieval loop #if __TBB_TASK_PRIORITY @@ -620,11 +629,10 @@ void custom_scheduler::local_wait_for_all( task& parent, task* t = receive_or_steal_task( __TBB_ISOLATION_ARG( parent.prefix().ref_count, isolation ) ); if ( !t ) goto done; - // The user can capture another the FPU settings to the context so the // cached data in the helper can be out-of-date and we cannot do fast // check. - cpu_ctl_helper.set_env( __TBB_CONTEXT_ARG1(t->prefix().context) ); + context_guard.set_ctx( __TBB_CONTEXT_ARG1(t->prefix().context) ); } // end of infinite stealing loop #if TBB_USE_EXCEPTIONS __TBB_ASSERT( false, "Must never get here" ); @@ -688,7 +696,7 @@ void custom_scheduler::local_wait_for_all( task& parent, task* // On Windows, FPU control settings changed in the helper destructor are not visible // outside a catch block. So restore the default settings manually before rethrowing // the exception. - cpu_ctl_helper.restore_default(); + context_guard.restore_default(); TbbRethrowException( pe ); } } diff --git a/src/tbb/src/tbb/dynamic_link.cpp b/src/tbb/src/tbb/dynamic_link.cpp index b51280d0..23221d47 100644 --- a/src/tbb/src/tbb/dynamic_link.cpp +++ b/src/tbb/src/tbb/dynamic_link.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "dynamic_link.h" @@ -54,7 +50,7 @@ #pragma weak dlclose #endif /* __TBB_WEAK_SYMBOLS_PRESENT && !__TBB_DYNAMIC_LOAD_ENABLED */ -#include "tbb/tbb_misc.h" +#include "tbb_misc.h" #define __USE_TBB_ATOMICS ( !(__linux__&&__ia64__) || __TBB_BUILD ) #define __USE_STATIC_DL_INIT ( !__ANDROID__ ) @@ -362,21 +358,20 @@ OPEN_INTERNAL_NAMESPACE in len -- Size of buffer. ret -- 0 -- Error occurred. > len -- Buffer too short, required size returned. - otherwise -- Ok, number of characters (not counting terminating null) written to - buffer. + otherwise -- Ok, number of characters (incl. terminating null) written to buffer. */ static size_t abs_path( char const * name, char * path, size_t len ) { - if ( !ap_data._len ) + if ( ap_data._len == 0 ) return 0; size_t name_len = strlen( name ); size_t full_len = name_len+ap_data._len; if ( full_len < len ) { - strncpy( path, ap_data._path, ap_data._len ); - memcpy( path+ap_data._len, name, name_len ); - path[full_len] = 0; + __TBB_ASSERT(ap_data._path[ap_data._len] == 0, NULL); + strcpy( path, ap_data._path ); + strcat( path, name ); } - return full_len; + return full_len+1; // +1 for null character } #endif // __TBB_DYNAMIC_LOAD_ENABLED @@ -512,37 +507,37 @@ OPEN_INTERNAL_NAMESPACE } dynamic_link_handle dynamic_load( const char* library, const dynamic_link_descriptor descriptors[], size_t required ) { - ::tbb::internal::suppress_unused_warning( library, descriptors, required ); - #if __TBB_DYNAMIC_LOAD_ENABLED + ::tbb::internal::suppress_unused_warning( library, descriptors, required ); +#if __TBB_DYNAMIC_LOAD_ENABLED - size_t const len = PATH_MAX + 1; - char path[ len ]; - size_t rc = abs_path( library, path, len ); - if ( 0 < rc && rc < len ) { + size_t const len = PATH_MAX + 1; + char path[ len ]; + size_t rc = abs_path( library, path, len ); + if ( 0 < rc && rc <= len ) { #if _WIN32 - // Prevent Windows from displaying silly message boxes if it fails to load library - // (e.g. because of MS runtime problems - one of those crazy manifest related ones) - UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS); + // Prevent Windows from displaying silly message boxes if it fails to load library + // (e.g. because of MS runtime problems - one of those crazy manifest related ones) + UINT prev_mode = SetErrorMode (SEM_FAILCRITICALERRORS); #endif /* _WIN32 */ - dynamic_link_handle library_handle = dlopen( path, RTLD_LAZY ); + dynamic_link_handle library_handle = dlopen( path, RTLD_LAZY ); #if _WIN32 - SetErrorMode (prev_mode); + SetErrorMode (prev_mode); #endif /* _WIN32 */ - if( library_handle ) { - if( !resolve_symbols( library_handle, descriptors, required ) ) { - // The loaded library does not contain all the expected entry points - dynamic_unlink( library_handle ); - library_handle = NULL; - } - } else - DYNAMIC_LINK_WARNING( dl_lib_not_found, path, dlerror() ); - return library_handle; - } else if ( rc>=len ) - DYNAMIC_LINK_WARNING( dl_buff_too_small ); - // rc == 0 means failing of init_ap_data so the warning has already been issued. + if( library_handle ) { + if( !resolve_symbols( library_handle, descriptors, required ) ) { + // The loaded library does not contain all the expected entry points + dynamic_unlink( library_handle ); + library_handle = NULL; + } + } else + DYNAMIC_LINK_WARNING( dl_lib_not_found, path, dlerror() ); + return library_handle; + } else if ( rc>len ) + DYNAMIC_LINK_WARNING( dl_buff_too_small ); + // rc == 0 means failing of init_ap_data so the warning has already been issued. - #endif /* __TBB_DYNAMIC_LOAD_ENABLED */ - return 0; +#endif /* __TBB_DYNAMIC_LOAD_ENABLED */ + return 0; } bool dynamic_link( const char* library, const dynamic_link_descriptor descriptors[], size_t required, dynamic_link_handle *handle, int flags ) { diff --git a/src/tbb/src/tbb/dynamic_link.h b/src/tbb/src/tbb/dynamic_link.h index cff51180..759f5807 100644 --- a/src/tbb/src/tbb/dynamic_link.h +++ b/src/tbb/src/tbb/dynamic_link.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_dynamic_link @@ -55,8 +51,10 @@ typedef void (*pointer_to_handler)(); // prevent warnings from some compilers (g++ 4.1) #if __TBB_WEAK_SYMBOLS_PRESENT #define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h), (pointer_to_handler)&s} +#define DLD_NOWEAK(s,h) {#s, (pointer_to_handler*)(void*)(&h), NULL} #else #define DLD(s,h) {#s, (pointer_to_handler*)(void*)(&h)} +#define DLD_NOWEAK(s,h) DLD(s,h) #endif /* __TBB_WEAK_SYMBOLS_PRESENT */ //! Association between a handler name and location of pointer to it. struct dynamic_link_descriptor { diff --git a/src/tbb/src/tbb/governor.cpp b/src/tbb/src/tbb/governor.cpp index b492e8ac..99516a96 100644 --- a/src/tbb/src/tbb/governor.cpp +++ b/src/tbb/src/tbb/governor.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include @@ -52,7 +48,7 @@ static __cilk_tbb_retcode (*watch_stack_handler)(struct __cilk_tbb_unwatch_thunk //! Table describing how to link the handlers. static const dynamic_link_descriptor CilkLinkTable[] = { - { "__cilkrts_watch_stack", (pointer_to_handler*)(void*)(&watch_stack_handler) } + DLD_NOWEAK(__cilkrts_watch_stack, watch_stack_handler) }; static atomic cilkrts_load_state; @@ -204,7 +200,7 @@ generic_scheduler* governor::init_scheduler( int num_threads, stack_size_type st return s; } -bool governor::terminate_scheduler( generic_scheduler* s, const task_scheduler_init* tsi_ptr, bool blocking ) { +bool governor::terminate_scheduler( generic_scheduler* s, bool blocking ) { bool ok = false; __TBB_ASSERT( is_set(s), "Attempt to terminate non-local scheduler instance" ); if (0 == --(s->my_ref_count)) { @@ -257,11 +253,8 @@ __cilk_tbb_retcode governor::stack_op_handler( __cilk_tbb_stack_op op, void* dat #else uintptr_t thread_id = uintptr_t(pthread_self()); #endif - #endif /* TBB_USE_ASSERT */ switch( op ) { - default: - __TBB_ASSERT( 0, "invalid op" ); case CILK_TBB_STACK_ADOPT: { __TBB_ASSERT( !current && s->my_cilk_state==generic_scheduler::cs_limbo || current==s && s->my_cilk_state==generic_scheduler::cs_running, "invalid adoption" ); @@ -289,7 +282,10 @@ __cilk_tbb_retcode governor::stack_op_handler( __cilk_tbb_stack_op op, void* dat #endif /* TBB_USE_ASSERT */ s->my_cilk_unwatch_thunk.routine = NULL; auto_terminate( s ); + break; } + default: + __TBB_ASSERT(0, "invalid op"); } return 0; } @@ -352,7 +348,7 @@ bool task_scheduler_init::internal_terminate( bool blocking ) { : vt & ~task_group_context::exact_exception; } #endif /* __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS */ - return governor::terminate_scheduler(s, this, blocking); + return governor::terminate_scheduler(s, blocking); } void task_scheduler_init::terminate() { diff --git a/src/tbb/src/tbb/governor.h b/src/tbb/src/tbb/governor.h index 92f98bb4..3a773bb4 100644 --- a/src/tbb/src/tbb/governor.h +++ b/src/tbb/src/tbb/governor.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_governor_H @@ -97,7 +93,7 @@ class governor { static generic_scheduler* init_scheduler_weak(); //! Processes scheduler termination request (possibly nested) in a master thread - static bool terminate_scheduler( generic_scheduler* s, const task_scheduler_init *tsi_ptr, bool blocking ); + static bool terminate_scheduler( generic_scheduler* s, bool blocking ); //! Register TBB scheduler instance in thread-local storage. static void sign_on( generic_scheduler* s ); diff --git a/src/tbb/src/tbb/ia32-masm/atomic_support.asm b/src/tbb/src/tbb/ia32-masm/atomic_support.asm index 3cb82b97..8974de17 100644 --- a/src/tbb/src/tbb/ia32-masm/atomic_support.asm +++ b/src/tbb/src/tbb/ia32-masm/atomic_support.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; .686 .model flat,c diff --git a/src/tbb/src/tbb/ia32-masm/itsx.asm b/src/tbb/src/tbb/ia32-masm/itsx.asm index 10e08357..b30015c6 100644 --- a/src/tbb/src/tbb/ia32-masm/itsx.asm +++ b/src/tbb/src/tbb/ia32-masm/itsx.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; .686 .model flat,c diff --git a/src/tbb/src/tbb/ia32-masm/lock_byte.asm b/src/tbb/src/tbb/ia32-masm/lock_byte.asm index daad5198..85de6ea5 100644 --- a/src/tbb/src/tbb/ia32-masm/lock_byte.asm +++ b/src/tbb/src/tbb/ia32-masm/lock_byte.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; ; DO NOT EDIT - AUTOMATICALLY GENERATED FROM .s FILE .686 diff --git a/src/tbb/src/tbb/ia64-gas/atomic_support.s b/src/tbb/src/tbb/ia64-gas/atomic_support.s index cf800dbc..320bedd2 100644 --- a/src/tbb/src/tbb/ia64-gas/atomic_support.s +++ b/src/tbb/src/tbb/ia64-gas/atomic_support.s @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh # 1 "" diff --git a/src/tbb/src/tbb/ia64-gas/ia64_misc.s b/src/tbb/src/tbb/ia64-gas/ia64_misc.s index b62ce393..0ee937f1 100644 --- a/src/tbb/src/tbb/ia64-gas/ia64_misc.s +++ b/src/tbb/src/tbb/ia64-gas/ia64_misc.s @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // RSE backing store pointer retrieval .section .text diff --git a/src/tbb/src/tbb/ia64-gas/lock_byte.s b/src/tbb/src/tbb/ia64-gas/lock_byte.s index 551ac330..947147a2 100644 --- a/src/tbb/src/tbb/ia64-gas/lock_byte.s +++ b/src/tbb/src/tbb/ia64-gas/lock_byte.s @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // Support for class TinyLock .section .text diff --git a/src/tbb/src/tbb/ia64-gas/log2.s b/src/tbb/src/tbb/ia64-gas/log2.s index d4ad9e2b..cec9e093 100644 --- a/src/tbb/src/tbb/ia64-gas/log2.s +++ b/src/tbb/src/tbb/ia64-gas/log2.s @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// .section .text .align 16 diff --git a/src/tbb/src/tbb/ia64-gas/pause.s b/src/tbb/src/tbb/ia64-gas/pause.s index a22b4203..683c4963 100644 --- a/src/tbb/src/tbb/ia64-gas/pause.s +++ b/src/tbb/src/tbb/ia64-gas/pause.s @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// .section .text .align 16 diff --git a/src/tbb/src/tbb/ibm_aix51/atomic_support.c b/src/tbb/src/tbb/ibm_aix51/atomic_support.c index 0aa08a1e..a43d768d 100644 --- a/src/tbb/src/tbb/ibm_aix51/atomic_support.c +++ b/src/tbb/src/tbb/ibm_aix51/atomic_support.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include diff --git a/src/tbb/src/tbb/index.html b/src/tbb/src/tbb/index.html index 9ba7f57e..f9cea0f3 100644 --- a/src/tbb/src/tbb/index.html +++ b/src/tbb/src/tbb/index.html @@ -21,7 +21,7 @@

Directories

Up to parent directory

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/tbb/intel64-masm/atomic_support.asm b/src/tbb/src/tbb/intel64-masm/atomic_support.asm index d3c7f698..0242775b 100644 --- a/src/tbb/src/tbb/intel64-masm/atomic_support.asm +++ b/src/tbb/src/tbb/intel64-masm/atomic_support.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; ; DO NOT EDIT - AUTOMATICALLY GENERATED FROM .s FILE .code diff --git a/src/tbb/src/tbb/intel64-masm/intel64_misc.asm b/src/tbb/src/tbb/intel64-masm/intel64_misc.asm index 4fe75152..ea26d582 100644 --- a/src/tbb/src/tbb/intel64-masm/intel64_misc.asm +++ b/src/tbb/src/tbb/intel64-masm/intel64_misc.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; .code ALIGN 8 diff --git a/src/tbb/src/tbb/intel64-masm/itsx.asm b/src/tbb/src/tbb/intel64-masm/itsx.asm index 1a5958fb..6e3ff711 100644 --- a/src/tbb/src/tbb/intel64-masm/itsx.asm +++ b/src/tbb/src/tbb/intel64-masm/itsx.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; .code ALIGN 8 diff --git a/src/tbb/src/tbb/intrusive_list.h b/src/tbb/src/tbb/intrusive_list.h index cb2190af..07ed1f31 100644 --- a/src/tbb/src/tbb/intrusive_list.h +++ b/src/tbb/src/tbb/intrusive_list.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_intrusive_list_H diff --git a/src/tbb/src/tbb/itt_notify.cpp b/src/tbb/src/tbb/itt_notify.cpp index 670c84d9..8e70ca43 100644 --- a/src/tbb/src/tbb/itt_notify.cpp +++ b/src/tbb/src/tbb/itt_notify.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if DO_ITT_NOTIFY @@ -59,9 +55,7 @@ int __TBB_load_ittnotify() { (__itt_group_id)(__itt_group_sync // prepare/cancel/acquired/releasing | __itt_group_thread // name threads | __itt_group_stitch // stack stitching -#if __TBB_CPF_BUILD | __itt_group_structure -#endif )); } @@ -90,6 +84,9 @@ namespace tbb { *SyncObj_Mailbox = _T("TBB Scheduler"), *SyncObj_TaskReturnList = _T("TBB Scheduler"), *SyncObj_TaskStream = _T("TBB Scheduler"), +#if __TBB_PREVIEW_CRITICAL_TASKS + *SyncObj_CriticalTaskStream = _T("TBB Scheduler"), +#endif *SyncObj_ContextsList = _T("TBB Scheduler") ; #endif /* DO_ITT_NOTIFY */ diff --git a/src/tbb/src/tbb/itt_notify.h b/src/tbb/src/tbb/itt_notify.h index 529437c8..89dd5c5a 100644 --- a/src/tbb/src/tbb/itt_notify.h +++ b/src/tbb/src/tbb/itt_notify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_ITT_NOTIFY @@ -85,6 +81,9 @@ namespace tbb { *SyncObj_Mailbox, *SyncObj_TaskReturnList, *SyncObj_TaskStream, +#if __TBB_PREVIEW_CRITICAL_TASKS + *SyncObj_CriticalTaskStream, +#endif *SyncObj_ContextsList ; @@ -108,6 +107,10 @@ namespace tbb { #define ITT_STACK(precond, name, obj) ((void)0) #endif /* !__TBB_TASK_GROUP_CONTEXT */ +#define ITT_TASK_GROUP(obj,name,parent) itt_make_task_group_v7(internal::ITT_DOMAIN_MAIN,(void*)(obj),ALGORITHM,(void*)(parent),(parent!=NULL) ? ALGORITHM : FLOW_NULL,name) +#define ITT_TASK_BEGIN(obj,name,id) itt_task_begin_v7(internal::ITT_DOMAIN_MAIN,(void*)(id),ALGORITHM,(void*)(obj),ALGORITHM,name) +#define ITT_TASK_END itt_task_end_v7(internal::ITT_DOMAIN_MAIN) + #else /* !DO_ITT_NOTIFY */ #define ITT_NOTIFY(name,obj) ((void)0) @@ -118,6 +121,10 @@ namespace tbb { #define ITT_STACK_CREATE(obj) ((void)0) #define ITT_STACK(precond, name, obj) ((void)0) +#define ITT_TASK_GROUP(type,name,parent) ((void)0) +#define ITT_TASK_BEGIN(type,name,id) ((void)0) +#define ITT_TASK_END ((void)0) + #endif /* !DO_ITT_NOTIFY */ namespace tbb { diff --git a/src/tbb/src/tbb/lin32-tbb-export.def b/src/tbb/src/tbb/lin32-tbb-export.def index d1f7c9c4..f3a9d56c 100644 --- a/src/tbb/src/tbb/lin32-tbb-export.def +++ b/src/tbb/src/tbb/lin32-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbb/lin32-tbb-export.lst b/src/tbb/src/tbb/lin32-tbb-export.lst index 10f27266..12a21b41 100644 --- a/src/tbb/src/tbb/lin32-tbb-export.lst +++ b/src/tbb/src/tbb/lin32-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -170,7 +166,6 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -#if __TBB_ITT_STRUCTURE_API __TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) __TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) __TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) @@ -178,7 +173,6 @@ __TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yN __TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) __TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) __TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) -#endif /* pipeline.cpp */ __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/lin64-tbb-export.def b/src/tbb/src/tbb/lin64-tbb-export.def index 1f3cac3a..192a78c7 100644 --- a/src/tbb/src/tbb/lin64-tbb-export.def +++ b/src/tbb/src/tbb/lin64-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbb/lin64-tbb-export.lst b/src/tbb/src/tbb/lin64-tbb-export.lst index f05e68b7..6f569839 100644 --- a/src/tbb/src/tbb/lin64-tbb-export.lst +++ b/src/tbb/src/tbb/lin64-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -165,7 +161,6 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) -#if __TBB_ITT_STRUCTURE_API __TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) __TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) __TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) @@ -173,7 +168,6 @@ __TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0 __TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) __TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) __TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) -#endif /* pipeline.cpp */ __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/lin64ipf-tbb-export.def b/src/tbb/src/tbb/lin64ipf-tbb-export.def index c2111c1e..23adff28 100644 --- a/src/tbb/src/tbb/lin64ipf-tbb-export.def +++ b/src/tbb/src/tbb/lin64ipf-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbb/lin64ipf-tbb-export.lst b/src/tbb/src/tbb/lin64ipf-tbb-export.lst index 49f8cee7..908f514f 100644 --- a/src/tbb/src/tbb/lin64ipf-tbb-export.lst +++ b/src/tbb/src/tbb/lin64ipf-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -166,6 +162,13 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) +__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) +__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) +__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) /* pipeline.cpp */ __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/mac32-tbb-export.def b/src/tbb/src/tbb/mac32-tbb-export.def index 5ed01b1a..3fcfa82b 100644 --- a/src/tbb/src/tbb/mac32-tbb-export.def +++ b/src/tbb/src/tbb/mac32-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define __TBB_SYMBOL( sym ) _##sym diff --git a/src/tbb/src/tbb/mac32-tbb-export.lst b/src/tbb/src/tbb/mac32-tbb-export.lst index 8ce79f2d..1ef5869b 100644 --- a/src/tbb/src/tbb/mac32-tbb-export.lst +++ b/src/tbb/src/tbb/mac32-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -24,7 +20,7 @@ Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when - necessary, depending on the indended usage. + necessary, depending on the intended usage. */ // cache_aligned_allocator.cpp @@ -191,6 +187,13 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) +__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) // pipeline.cpp __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/mac64-tbb-export.def b/src/tbb/src/tbb/mac64-tbb-export.def index e2c03705..a83d48ba 100644 --- a/src/tbb/src/tbb/mac64-tbb-export.def +++ b/src/tbb/src/tbb/mac64-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define __TBB_SYMBOL( sym ) _##sym diff --git a/src/tbb/src/tbb/mac64-tbb-export.lst b/src/tbb/src/tbb/mac64-tbb-export.lst index 67d3676d..c2d82d97 100644 --- a/src/tbb/src/tbb/mac64-tbb-export.lst +++ b/src/tbb/src/tbb/mac64-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -24,7 +20,7 @@ Sometimes macOS* requires leading underscore (e. g. in export list file), but sometimes not (e. g. when searching symbol in a dynamic library via dlsym()). Symbols in this file SHOULD be listed WITHOUT one leading underscore. __TBB_SYMBOL macro should add underscore when - necessary, depending on the indended usage. + necessary, depending on the intended usage. */ // cache_aligned_allocator.cpp @@ -188,6 +184,13 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) +__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) +__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) +__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) // pipeline.cpp __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/mailbox.h b/src/tbb/src/tbb/mailbox.h index f651589e..bd02355e 100644 --- a/src/tbb/src/tbb/mailbox.h +++ b/src/tbb/src/tbb/mailbox.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_mailbox_H diff --git a/src/tbb/src/tbb/market.cpp b/src/tbb/src/tbb/market.cpp index 3b6efc63..98f3a022 100644 --- a/src/tbb/src/tbb/market.cpp +++ b/src/tbb/src/tbb/market.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_stddef.h" @@ -190,7 +186,7 @@ bool market::release ( bool is_public, bool blocking_terminate ) { // Theoretically, new private references to the market can be added during waiting making it potentially // endless. // TODO: revise why the weak scheduler needs market's pointer and try to remove this wait. - // Note that the market should know about its schedulers for cancelation/exception/priority propagation, + // Note that the market should know about its schedulers for cancellation/exception/priority propagation, // see e.g. task_group_context::cancel_group_execution() while ( __TBB_load_with_acquire( my_public_ref_count ) == 1 && __TBB_load_with_acquire( my_ref_count ) > 1 ) __TBB_Yield(); @@ -329,7 +325,7 @@ void market::try_destroy_arena ( arena* a, uintptr_t aba_epoch ) { assert_market_valid(); #if __TBB_TASK_PRIORITY // scan all priority levels, not only in [my_global_bottom_priority;my_global_top_priority] - // range, because arena to be destoyed can have no outstanding request for workers + // range, because arena to be destroyed can have no outstanding request for workers for ( int p = num_priority_levels-1; p >= 0; --p ) { priority_level_info &pl = my_priority_levels[p]; arena_list_type &my_arenas = pl.arenas; @@ -360,10 +356,10 @@ void market::try_destroy_arena ( arena* a, uintptr_t aba_epoch ) { } /** This method must be invoked under my_arenas_list_mutex. **/ -arena* market::arena_in_need ( arena_list_type &arenas, arena *&next ) { +arena* market::arena_in_need ( arena_list_type &arenas, arena *hint ) { if ( arenas.empty() ) return NULL; - arena_list_type::iterator it = next; + arena_list_type::iterator it = hint; __TBB_ASSERT( it != arenas.end(), NULL ); do { arena& a = *it; @@ -375,11 +371,9 @@ arena* market::arena_in_need ( arena_list_type &arenas, arena *&next ) { #endif ) { a.my_references += arena::ref_worker; - as_atomic(next) = &*it; // a subject for innocent data race under the reader lock - // TODO: rework global round robin policy to local or random to avoid this write return &a; } - } while ( it != next ); + } while ( it != hint ); return NULL; } @@ -415,6 +409,16 @@ int market::update_allotment ( arena_list_type& arenas, int workers_demand, int return assigned; } +/** This method must be invoked under my_arenas_list_mutex. **/ +bool market::is_arena_in_list( arena_list_type &arenas, arena *a ) { + if ( a ) { + for ( arena_list_type::iterator it = arenas.begin(); it != arenas.end(); ++it ) + if ( a == &*it ) + return true; + } + return false; +} + #if __TBB_TASK_PRIORITY inline void market::update_global_top_priority ( intptr_t newPriority ) { GATHER_STATISTIC( ++governor::local_scheduler_if_initialized()->my_counters.market_prio_switches ); @@ -432,21 +436,29 @@ inline void market::reset_global_priority () { update_global_top_priority(normalized_normal_priority); } -arena* market::arena_in_need ( arena* prev_arena ) -{ - suppress_unused_warning(prev_arena); +arena* market::arena_in_need ( arena* prev_arena ) { if( as_atomic(my_total_demand) <= 0 ) return NULL; arenas_list_mutex_type::scoped_lock lock(my_arenas_list_mutex, /*is_writer=*/false); assert_market_valid(); int p = my_global_top_priority; arena *a = NULL; - do { - priority_level_info &pl = my_priority_levels[p]; + + // Checks if arena is alive or not + if ( is_arena_in_list( my_priority_levels[p].arenas, prev_arena ) ) { + a = arena_in_need( my_priority_levels[p].arenas, prev_arena ); + } + + while ( !a && p >= my_global_bottom_priority ) { + priority_level_info &pl = my_priority_levels[p--]; a = arena_in_need( pl.arenas, pl.next_arena ); + if ( a ) { + as_atomic(pl.next_arena) = a; // a subject for innocent data race under the reader lock + // TODO: rework global round robin policy to local or random to avoid this write + } // TODO: When refactoring task priority code, take into consideration the // __TBB_TRACK_PRIORITY_LEVEL_SATURATION sections from earlier versions of TBB - } while ( !a && --p >= my_global_bottom_priority ); + } return a; } @@ -681,43 +693,23 @@ void market::adjust_demand ( arena& a, int delta ) { void market::process( job& j ) { generic_scheduler& s = static_cast(j); - arena *a = NULL; + // s.my_arena can be dead. Don't access it until arena_in_need is called + arena *a = s.my_arena; __TBB_ASSERT( governor::is_set(&s), NULL ); - enum { - query_interval = 1000, - first_interval = 1 - }; - for(int i = first_interval; ; i--) { - while ( (a = arena_in_need(a)) ) - { + + for (int i = 0; i < 2; ++i) { + while ( (a = arena_in_need(a)) ) { a->process(s); - i = first_interval; + a = NULL; // to avoid double checks in arena_in_need(arena*) for the same priority level } // Workers leave market because there is no arena in need. It can happen earlier than // adjust_job_count_estimate() decreases my_slack and RML can put this thread to sleep. // It might result in a busy-loop checking for my_slack<0 and calling this method instantly. - // first_interval>0 and the yield refines this spinning. - if( i > 0 ) + // the yield refines this spinning. + if ( !i ) __TBB_Yield(); - else -#if !__TBB_SLEEP_PERMISSION - break; -#else - { // i == 0 -#if __TBB_TASK_PRIORITY - arena_list_type &al = my_priority_levels[my_global_top_priority].arenas; -#else /* __TBB_TASK_PRIORITY */ - arena_list_type &al = my_arenas; -#endif /* __TBB_TASK_PRIORITY */ - if( al.empty() ) // races if any are innocent TODO: replace by an RML query interface - break; // no arenas left, perhaps going to shut down - if( the_global_observer_list.ask_permission_to_leave() ) - break; // go sleep - __TBB_Yield(); - i = query_interval; - } -#endif// !__TBB_SLEEP_PERMISSION } + GATHER_STATISTIC( ++s.my_counters.market_roundtrips ); } diff --git a/src/tbb/src/tbb/market.h b/src/tbb/src/tbb/market.h index 39d12d5e..ee4e9038 100644 --- a/src/tbb/src/tbb/market.h +++ b/src/tbb/src/tbb/market.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_market_H @@ -217,6 +213,7 @@ class market : no_copy, rml::tbb_client { update_allotment( my_arenas, my_total_demand, (int)my_num_workers_soft_limit ); } + // TODO: consider to rewrite the code with is_arena_in_list function //! Returns next arena that needs more workers, or NULL. arena* arena_in_need (arena*) { if(__TBB_load_with_acquire(my_total_demand) <= 0) @@ -234,10 +231,12 @@ class market : no_copy, rml::tbb_client { void remove_arena_from_list ( arena& a ); - arena* arena_in_need ( arena_list_type &arenas, arena *&next ); + arena* arena_in_need ( arena_list_type &arenas, arena *hint ); static int update_allotment ( arena_list_type& arenas, int total_demand, int max_workers ); + bool is_arena_in_list( arena_list_type &arenas, arena *a ); + //////////////////////////////////////////////////////////////////////////////// // Implementation of rml::tbb_client interface methods diff --git a/src/tbb/src/tbb/mutex.cpp b/src/tbb/src/tbb/mutex.cpp index fd02d260..677aa675 100644 --- a/src/tbb/src/tbb/mutex.cpp +++ b/src/tbb/src/tbb/mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if _WIN32||_WIN64 diff --git a/src/tbb/src/tbb/observer_proxy.cpp b/src/tbb/src/tbb/observer_proxy.cpp index 2a9b2bf4..a214155e 100644 --- a/src/tbb/src/tbb/observer_proxy.cpp +++ b/src/tbb/src/tbb/observer_proxy.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -47,7 +43,7 @@ struct check_observer_proxy_count { static check_observer_proxy_count the_check_observer_proxy_count; #endif /* TBB_USE_ASSERT */ -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER interface6::task_scheduler_observer* observer_proxy::get_v6_observer() { if(my_version != 6) return NULL; return static_cast(my_observer); @@ -295,64 +291,6 @@ void observer_list::do_notify_exit_observers( observer_proxy* last, bool worker } } -#if __TBB_SLEEP_PERMISSION -bool observer_list::ask_permission_to_leave() { - __TBB_ASSERT( this == &the_global_observer_list, "This method cannot be used on lists of arena observers" ); - if( !my_head ) return true; - // Pointer p marches though the list - observer_proxy *p = NULL, *prev = NULL; - bool result = true; - while( result ) { - task_scheduler_observer* tso = NULL; - // Hold lock on list only long enough to advance to the next proxy in the list. - { - scoped_lock lock(mutex(), /*is_writer=*/false); - do { - if( p ) { - // We were already processing the list. - observer_proxy* q = p->my_next; - // read next, remove the previous reference - if( p == prev ) - remove_ref_fast(prev); // sets prev to NULL if successful - if( q ) p = q; - else { - // Reached the end of the list. - if( prev ) { - lock.release(); - remove_ref(prev); - } - return result; - } - } else { - // Starting pass through the list - p = my_head; - if( !p ) - return result; - } - tso = p->get_v6_observer(); - } while( !tso ); - ++p->my_ref_count; - ++tso->my_busy_count; - } - __TBB_ASSERT( !prev || p!=prev, NULL ); - // Release the proxy pinned before p - if( prev ) - remove_ref(prev); - // Do not hold any locks on the list while calling user's code. - // Do not intercept any exceptions that may escape the callback so that - // they are either handled by the TBB scheduler or passed to the debugger. - result = tso->may_sleep(); - __TBB_ASSERT(p->my_ref_count, NULL); - intptr_t bc = --tso->my_busy_count; - __TBB_ASSERT_EX( bc>=0, "my_busy_count underflowed" ); - prev = p; - } - if( prev ) - remove_ref(prev); - return result; -} -#endif//__TBB_SLEEP_PERMISSION - void task_scheduler_observer_v3::observe( bool enable ) { if( enable ) { if( !my_proxy ) { @@ -369,7 +307,7 @@ void task_scheduler_observer_v3::observe( bool enable ) { a->initialize(); my_proxy->my_list = &a->my_arena->my_observers; } else { - if( !s ) + if( !(s && s->my_arena) ) s = governor::init_scheduler( task_scheduler_init::automatic, 0, true ); __TBB_ASSERT( __TBB_InitOnce::initialization_done(), NULL ); __TBB_ASSERT( s && s->my_arena, NULL ); diff --git a/src/tbb/src/tbb/observer_proxy.h b/src/tbb/src/tbb/observer_proxy.h index 7f2c4e39..2a0fb5bf 100644 --- a/src/tbb/src/tbb/observer_proxy.h +++ b/src/tbb/src/tbb/observer_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_observer_proxy_H @@ -92,9 +88,6 @@ class observer_list { //! Call exit notifications on last and observers added before it. inline void notify_exit_observers( observer_proxy*& last, bool worker ); - - //! Call may_sleep callbacks to ask for permission for a worker thread to leave market - bool ask_permission_to_leave(); }; // class observer_list //! Wrapper for an observer object @@ -122,7 +115,7 @@ class observer_proxy { //! Version char my_version; -#if __TBB_ARENA_OBSERVER || __TBB_SLEEP_PERMISSION +#if __TBB_ARENA_OBSERVER interface6::task_scheduler_observer* get_v6_observer(); #endif #if __TBB_ARENA_OBSERVER diff --git a/src/tbb/src/tbb/pipeline.cpp b/src/tbb/src/tbb/pipeline.cpp index 96a0aeea..b3cd441c 100644 --- a/src/tbb/src/tbb/pipeline.cpp +++ b/src/tbb/src/tbb/pipeline.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/pipeline.h" @@ -169,8 +165,8 @@ class input_buffer : no_copy { //! Note that processing of a token is finished. /** Fires up processing of the next token, if processing was deferred. */ - // Using template to avoid explicit dependency on stage_task - // this is only called for serial filters, and is the reason for the + // Uses template to avoid explicit dependency on stage_task. + // This is only called for serial filters, and is the reason for the // advance parameter in return_item (we're incrementing low_token here.) // Non-TBF serial stages don't advance the token at the start because the presence // of the current token in the buffer keeps another stage from being spawned. @@ -206,8 +202,8 @@ class input_buffer : no_copy { } #endif - //! return an item, invalidate the queued item, but only advance if advance - // advance == true for parallel filters. If the filter is serial, leave the + //! return an item, invalidate the queued item, but only advance if the filter + // is parallel (as indicated by advance == true). If the filter is serial, leave the // item in the buffer to keep another stage from being spawned. bool return_item(task_info& info, bool advance) { spin_mutex::scoped_lock lock( array_mutex ); @@ -425,10 +421,27 @@ class pipeline_root_task: public task { { task_info info; info.reset(); - if( current_filter->my_input_buffer->return_item(info, !current_filter->is_serial()) ) { - set_ref_count(1); + task* bypass = NULL; + int refcnt = 0; + task_list list; + // No new tokens are created; it's OK to process all waiting tokens. + // If the filter is serial, the second call to return_item will return false. + while( current_filter->my_input_buffer->return_item(info, !current_filter->is_serial()) ) { + task* t = new( allocate_child() ) stage_task( my_pipeline, current_filter, info ); + if( ++refcnt == 1 ) + bypass = t; + else // there's more than one task + list.push_back(*t); + // TODO: limit the list size (to arena size?) to spawn tasks sooner + __TBB_ASSERT( refcnt <= int(my_pipeline.token_counter), "token counting error" ); + info.reset(); + } + if( refcnt ) { + set_ref_count( refcnt ); + if( refcnt > 1 ) + spawn(list); recycle_as_continuation(); - return new( allocate_child() ) stage_task( my_pipeline, current_filter, info); + return bypass; } current_filter = current_filter->next_segment; if( !current_filter ) { @@ -565,9 +578,7 @@ void pipeline::add_filter( filter& filter_ ) { filter_end->next_filter_in_pipeline = &filter_; filter_.next_filter_in_pipeline = NULL; filter_end = &filter_; - } - else - { + } else { if( !filter_end ) filter_end = reinterpret_cast(&filter_list); @@ -580,15 +591,13 @@ void pipeline::add_filter( filter& filter_ ) { if( filter_.is_bound() ) has_thread_bound_filters = true; filter_.my_input_buffer = new internal::input_buffer( filter_.is_ordered(), filter_.is_bound() ); - } - else { + } else { if(filter_.prev_filter_in_pipeline) { if(filter_.prev_filter_in_pipeline->is_bound()) { // successors to bound filters must have an input_buffer filter_.my_input_buffer = new internal::input_buffer( /*is_ordered*/false, false ); } - } - else { // input filter + } else { // input filter if(filter_.object_may_be_null() ) { //TODO: buffer only needed to hold TLS; could improve filter_.my_input_buffer = new internal::input_buffer( /*is_ordered*/false, false ); @@ -696,14 +705,12 @@ filter::~filter() { } } -void -filter::set_end_of_input() { +void filter::set_end_of_input() { __TBB_ASSERT(my_input_buffer, NULL); __TBB_ASSERT(object_may_be_null(), NULL); if(is_serial()) { my_pipeline->end_of_input = true; - } - else { + } else { __TBB_ASSERT(my_input_buffer->end_of_input_tls_allocated, NULL); my_input_buffer->set_my_tls_end_of_input(); } diff --git a/src/tbb/src/tbb/private_server.cpp b/src/tbb/src/tbb/private_server.cpp index d9be168c..4f3b3b78 100644 --- a/src/tbb/src/tbb/private_server.cpp +++ b/src/tbb/src/tbb/private_server.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,14 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ -#include "rml_tbb.h" -#include "../server/thread_monitor.h" +#include "../rml/include/rml_tbb.h" +#include "../rml/server/thread_monitor.h" #include "tbb/atomic.h" #include "tbb/cache_aligned_allocator.h" #include "scheduler_common.h" @@ -275,6 +271,7 @@ void private_worker::run() { // Check/set the invariant for sleeping if( my_state!=st_quit && my_server.try_insert_in_asleep_list(*this) ) { my_thread_monitor.commit_wait(c); + __TBB_ASSERT( my_state==st_quit || !my_next, "Thread monitor missed a spurious wakeup?" ); my_server.propagate_chain_reaction(); } else { // Invariant broken @@ -310,8 +307,10 @@ inline void private_worker::wake_or_launch() { release_handle(my_handle, governor::does_client_join_workers(my_client)); } } - else + else { + __TBB_ASSERT( !my_next, "Should not wake a thread while it's still in asleep list" ); my_thread_monitor.notify(); + } } //------------------------------------------------------------------------ @@ -390,8 +389,11 @@ void private_server::wake_some( int additional_slack ) { } } done: - while( w>wakee ) - (*--w)->wake_or_launch(); + while( w>wakee ) { + private_worker* ww = *--w; + ww->my_next = NULL; + ww->wake_or_launch(); + } } void private_server::adjust_job_count_estimate( int delta ) { diff --git a/src/tbb/src/tbb/queuing_mutex.cpp b/src/tbb/src/tbb/queuing_mutex.cpp index 0071f8b9..1aa11def 100644 --- a/src/tbb/src/tbb/queuing_mutex.cpp +++ b/src/tbb/src/tbb/queuing_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/queuing_mutex.h" diff --git a/src/tbb/src/tbb/queuing_rw_mutex.cpp b/src/tbb/src/tbb/queuing_rw_mutex.cpp index f19a86bd..ce751571 100644 --- a/src/tbb/src/tbb/queuing_rw_mutex.cpp +++ b/src/tbb/src/tbb/queuing_rw_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ /** Before making any changes in the implementation, please emulate algorithmic changes @@ -363,7 +359,7 @@ void queuing_rw_mutex::scoped_lock::release( ) bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() { - __TBB_ASSERT( my_state==STATE_WRITER, "no sense to downgrade a reader" ); + if ( my_state == STATE_ACTIVEREADER ) return true; // Already a reader ITT_NOTIFY(sync_releasing, my_mutex); my_state = STATE_READER; @@ -390,7 +386,7 @@ bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() bool queuing_rw_mutex::scoped_lock::upgrade_to_writer() { - __TBB_ASSERT( my_state==STATE_ACTIVEREADER, "only active reader can be upgraded" ); + if ( my_state == STATE_WRITER ) return true; // Already a writer queuing_rw_mutex::scoped_lock * tmp; queuing_rw_mutex::scoped_lock * me = this; diff --git a/src/tbb/src/tbb/reader_writer_lock.cpp b/src/tbb/src/tbb/reader_writer_lock.cpp index 770ab25d..6fae2346 100644 --- a/src/tbb/src/tbb/reader_writer_lock.cpp +++ b/src/tbb/src/tbb/reader_writer_lock.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/reader_writer_lock.h" diff --git a/src/tbb/src/tbb/recursive_mutex.cpp b/src/tbb/src/tbb/recursive_mutex.cpp index 7d33202c..18565bc9 100644 --- a/src/tbb/src/tbb/recursive_mutex.cpp +++ b/src/tbb/src/tbb/recursive_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/recursive_mutex.h" diff --git a/src/tbb/src/tbb/scheduler.cpp b/src/tbb/src/tbb/scheduler.cpp index adb01c80..9b08d403 100644 --- a/src/tbb/src/tbb/scheduler.cpp +++ b/src/tbb/src/tbb/scheduler.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "custom_scheduler.h" @@ -98,6 +94,9 @@ generic_scheduler::generic_scheduler( market& m ) __TBB_ASSERT( governor::is_set(NULL), "scheduler is already initialized for this thread" ); my_innermost_running_task = my_dummy_task = &allocate_task( sizeof(task), __TBB_CONTEXT_ARG(NULL, &the_dummy_context) ); +#if __TBB_PREVIEW_CRITICAL_TASKS + my_properties.has_taken_critical_task = false; +#endif my_properties.outermost = true; #if __TBB_TASK_PRIORITY my_ref_top_priority = &m.my_global_top_priority; @@ -260,6 +259,9 @@ void generic_scheduler::cleanup_local_context_list () { void generic_scheduler::free_scheduler() { __TBB_ASSERT( !my_arena_slot, NULL ); +#if __TBB_PREVIEW_CRITICAL_TASKS + __TBB_ASSERT( !my_properties.has_taken_critical_task, "Critical tasks miscount." ); +#endif #if __TBB_TASK_GROUP_CONTEXT cleanup_local_context_list(); #endif /* __TBB_TASK_GROUP_CONTEXT */ @@ -591,6 +593,22 @@ inline task* generic_scheduler::prepare_for_spawning( task* t ) { return t; } +#if __TBB_PREVIEW_CRITICAL_TASKS +bool generic_scheduler::handled_as_critical( task& t ) { + if( !internal::is_critical( t ) ) + return false; +#if __TBB_TASK_ISOLATION + t.prefix().isolation = my_innermost_running_task->prefix().isolation; +#endif + ITT_NOTIFY(sync_releasing, &my_arena->my_critical_task_stream); + __TBB_ASSERT( my_arena, "Must be attached to the arena." ); + __TBB_ASSERT( my_arena_slot, "Must occupy a slot in the attached arena" ); + my_arena->my_critical_task_stream.push( + &t, 0, tbb::internal::subsequent_lane_selector(my_arena_slot->hint_for_critical) ); + return true; +} +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ + /** Conceptually, this method should be a member of class scheduler. But doing so would force us to publish class scheduler in the headers. */ void generic_scheduler::local_spawn( task* first, task*& next ) { @@ -609,9 +627,16 @@ void generic_scheduler::local_spawn( task* first, task*& next ) { // change the implementation of the task_list class). But since such changes // may affect the binary compatibility, we postpone them for a while. #endif - size_t T = prepare_task_pool( 1 ); - my_arena_slot->task_pool_ptr[T] = prepare_for_spawning( first ); - commit_spawned_tasks( T + 1 ); +#if __TBB_PREVIEW_CRITICAL_TASKS + if( !handled_as_critical( *first ) ) +#endif + { + size_t T = prepare_task_pool( 1 ); + my_arena_slot->task_pool_ptr[T] = prepare_for_spawning( first ); + commit_spawned_tasks( T + 1 ); + if ( !is_task_pool_published() ) + publish_task_pool(); + } } else { // Task list is being spawned @@ -634,17 +659,21 @@ void generic_scheduler::local_spawn( task* first, task*& next ) { // So milk it while it is alive. bool end = &t->prefix().next == &next; t_next = t->prefix().next; - tasks.push_back( prepare_for_spawning(t) ); +#if __TBB_PREVIEW_CRITICAL_TASKS + if( !handled_as_critical( *t ) ) +#endif + tasks.push_back( prepare_for_spawning(t) ); if( end ) break; } - size_t num_tasks = tasks.size(); - size_t T = prepare_task_pool( num_tasks ); - tasks.copy_memory( my_arena_slot->task_pool_ptr + T ); - commit_spawned_tasks( T + num_tasks ); + if( size_t num_tasks = tasks.size() ) { + size_t T = prepare_task_pool( num_tasks ); + tasks.copy_memory( my_arena_slot->task_pool_ptr + T ); + commit_spawned_tasks( T + num_tasks ); + if ( !is_task_pool_published() ) + publish_task_pool(); + } } - if ( !is_task_pool_published() ) - publish_task_pool(); my_arena->advertise_new_work(); assert_task_pool_valid(); } @@ -1039,7 +1068,42 @@ inline task* generic_scheduler::get_task( __TBB_ISOLATION_EXPR( isolation_tag is return result; } // generic_scheduler::get_task -task* generic_scheduler::steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_slot, isolation_tag isolation ) ) { +task* generic_scheduler::steal_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) ) { + // Try to steal a task from a random victim. + size_t k = my_random.get() % (my_arena->my_limit-1); + arena_slot* victim = &my_arena->my_slots[k]; + // The following condition excludes the master that might have + // already taken our previous place in the arena from the list . + // of potential victims. But since such a situation can take + // place only in case of significant oversubscription, keeping + // the checks simple seems to be preferable to complicating the code. + if( k >= my_arena_index ) + ++victim; // Adjusts random distribution to exclude self + task **pool = victim->task_pool; + task *t = NULL; + if( pool == EmptyTaskPool || !(t = steal_task_from( __TBB_ISOLATION_ARG(*victim, isolation) )) ) + return NULL; + if( is_proxy(*t) ) { + task_proxy &tp = *(task_proxy*)t; + t = tp.extract_task(); + if ( !t ) { + // Proxy was empty, so it's our responsibility to free it + free_task(tp); + return NULL; + } + GATHER_STATISTIC( ++my_counters.proxies_stolen ); + } + t->prefix().extra_state |= es_task_is_stolen; + if( is_version_3_task(*t) ) { + my_innermost_running_task = t; + t->prefix().owner = this; + t->note_affinity( my_affinity_id ); + } + GATHER_STATISTIC( ++my_counters.steals_committed ); + return t; +} + +task* generic_scheduler::steal_task_from( __TBB_ISOLATION_ARG( arena_slot& victim_slot, isolation_tag isolation ) ) { task** victim_pool = lock_task_pool( &victim_slot ); if ( !victim_pool ) return NULL; @@ -1072,7 +1136,7 @@ task* generic_scheduler::steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_slo break; GATHER_STATISTIC( ++my_counters.proxies_bypassed ); } - // The task cannot be executed either due to isolation or proxy contraints. + // The task cannot be executed either due to isolation or proxy constraints. result = NULL; tasks_omitted = true; } else if ( !tasks_omitted ) { @@ -1104,6 +1168,31 @@ task* generic_scheduler::steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_slo return result; } +#if __TBB_PREVIEW_CRITICAL_TASKS +// Retrieves critical task respecting isolation level, if provided. The rule is: +// 1) If no outer critical task and no isolation => take any critical task +// 2) If working on an outer critical task and no isolation => cannot take any critical task +// 3) If no outer critical task but isolated => respect isolation +// 4) If working on an outer critical task and isolated => respect isolation +task* generic_scheduler::get_critical_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) ) { + __TBB_ASSERT( my_arena && my_arena_slot, "Must be attached to arena" ); + if( my_arena->my_critical_task_stream.empty(0) ) + return NULL; + task* critical_task = NULL; + // To keep some LIFO-ness, start search with the lane that was used during push operation. + unsigned& start_lane = my_arena_slot->hint_for_critical; +#if __TBB_TASK_ISOLATION + if( isolation != no_isolation ) { + critical_task = my_arena->my_critical_task_stream.pop_specific( 0, start_lane, isolation ); + } else +#endif + if( !my_properties.has_taken_critical_task ) { + critical_task = my_arena->my_critical_task_stream.pop( 0, preceding_lane_selector(start_lane) ); + } + return critical_task; +} +#endif + task* generic_scheduler::get_mailbox_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) ) { __TBB_ASSERT( my_affinity_id>0, "not in arena" ); while ( task_proxy* const tp = my_inbox.pop( __TBB_ISOLATION_EXPR( isolation ) ) ) { diff --git a/src/tbb/src/tbb/scheduler.h b/src/tbb/src/tbb/scheduler.h index 8dda929f..ace18dba 100644 --- a/src/tbb/src/tbb/scheduler.h +++ b/src/tbb/src/tbb/scheduler.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_scheduler_H @@ -55,8 +51,16 @@ struct scheduler_properties { //! Indicates that a scheduler is on outermost level. /** Note that the explicit execute method will set this property. **/ bool outermost : 1; +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Indicates that a scheduler is in the process of executing critical task(s). + bool has_taken_critical_task : 1; + + //! Reserved bits + unsigned char : 5; +#else //! Reserved bits unsigned char : 6; +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ }; struct scheduler_state { @@ -72,6 +76,7 @@ struct scheduler_state { //! Innermost task whose task::execute() is running. A dummy task on the outermost level. task* my_innermost_running_task; + mail_inbox my_inbox; //! The mailbox id assigned to this scheduler. @@ -110,7 +115,7 @@ struct scheduler_state { the thread that creates the instance. Class generic_scheduler is an abstract base class that contains most of the scheduler, - except for tweaks specific to processors and tools (e.g. VTune). + except for tweaks specific to processors and tools (e.g. VTune(TM) Performance Tools). The derived template class custom_scheduler fills in the tweaks. */ class generic_scheduler: public scheduler , public ::rml::job @@ -122,7 +127,11 @@ class generic_scheduler: public scheduler static const size_t quick_task_size = 256-task_prefix_reservation_size; static bool is_version_3_task( task& t ) { +#if __TBB_PREVIEW_CRITICAL_TASKS + return (t.prefix().extra_state & 0x7)>=0x1; +#else return (t.prefix().extra_state & 0x0F)>=0x1; +#endif } //! Position in the call stack specifying its maximal filling when stealing is still allowed @@ -261,8 +270,20 @@ class generic_scheduler: public scheduler return t.prefix().extra_state==es_task_proxy; } + //! Attempts to steal a task from a randomly chosen thread/scheduler + task* steal_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) ); + //! Steal task from another scheduler's ready pool. - task* steal_task( __TBB_ISOLATION_ARG( arena_slot& victim_arena_slot, isolation_tag isolation ) ); + task* steal_task_from( __TBB_ISOLATION_ARG( arena_slot& victim_arena_slot, isolation_tag isolation ) ); + +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Tries to find critical task in critical task stream + task* get_critical_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) ); + + //! Pushes task to critical task stream if it appears to be such task and returns + //! true. Otherwise does nothing and returns false. + bool handled_as_critical( task& t ); +#endif /** Initial size of the task deque sufficient to serve without reallocation 4 nested parallel_for calls with iteration space of 65535 grains each. **/ @@ -691,37 +712,99 @@ inline void generic_scheduler::offload_task ( task& t, intptr_t /*priority*/ ) { } #endif /* __TBB_TASK_PRIORITY */ +#if __TBB_PREVIEW_CRITICAL_TASKS +class critical_task_count_guard : internal::no_copy { +public: + critical_task_count_guard(scheduler_properties& properties, task& t) + : my_properties(properties), + my_original_critical_task_state(properties.has_taken_critical_task) { + my_properties.has_taken_critical_task |= internal::is_critical(t); + } + ~critical_task_count_guard() { + my_properties.has_taken_critical_task = my_original_critical_task_state; + } +private: + scheduler_properties& my_properties; + bool my_original_critical_task_state; +}; +#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ + +#if __TBB_FP_CONTEXT || __TBB_TASK_GROUP_CONTEXT +//! Helper class for tracking floating point context and task group context switches +/** Assuming presence of an itt collector, in addition to keeping track of floating + point context, this class emits itt events to indicate begin and end of task group + context execution **/ +template +class context_guard_helper { +#if __TBB_TASK_GROUP_CONTEXT + const task_group_context *curr_ctx; +#endif #if __TBB_FP_CONTEXT -class cpu_ctl_env_helper { cpu_ctl_env guard_cpu_ctl_env; cpu_ctl_env curr_cpu_ctl_env; +#endif public: - cpu_ctl_env_helper() { + context_guard_helper() +#if __TBB_TASK_GROUP_CONTEXT + : curr_ctx(NULL) +#endif + { +#if __TBB_FP_CONTEXT guard_cpu_ctl_env.get_env(); curr_cpu_ctl_env = guard_cpu_ctl_env; +#endif } - ~cpu_ctl_env_helper() { + ~context_guard_helper() { +#if __TBB_FP_CONTEXT if ( curr_cpu_ctl_env != guard_cpu_ctl_env ) guard_cpu_ctl_env.set_env(); +#endif +#if __TBB_TASK_GROUP_CONTEXT + if (report_tasks && curr_ctx) + ITT_TASK_END; +#endif } - void set_env( const task_group_context *ctx ) { + void set_ctx( const task_group_context *ctx ) { generic_scheduler::assert_context_valid(ctx); +#if __TBB_FP_CONTEXT const cpu_ctl_env &ctl = *punned_cast(&ctx->my_cpu_ctl_env); - if ( ctl != curr_cpu_ctl_env ) { - curr_cpu_ctl_env = ctl; - curr_cpu_ctl_env.set_env(); +#endif +#if __TBB_TASK_GROUP_CONTEXT + if(ctx != curr_ctx) { +#endif +#if __TBB_FP_CONTEXT + if ( ctl != curr_cpu_ctl_env ) { + curr_cpu_ctl_env = ctl; + curr_cpu_ctl_env.set_env(); + } +#endif +#if __TBB_TASK_GROUP_CONTEXT + // if task group context was active, report end of current execution frame. + if (report_tasks) { + if (curr_ctx) + ITT_TASK_END; + // reporting begin of new task group context execution frame. + // using address of task group context object to group tasks (parent). + // id of task execution frame is NULL and reserved for future use. + ITT_TASK_BEGIN(ctx,ctx->my_name,NULL); + curr_ctx = ctx; + } } +#endif } void restore_default() { +#if __TBB_FP_CONTEXT if ( curr_cpu_ctl_env != guard_cpu_ctl_env ) { guard_cpu_ctl_env.set_env(); curr_cpu_ctl_env = guard_cpu_ctl_env; } +#endif } }; #else -struct cpu_ctl_env_helper { - void set_env( __TBB_CONTEXT_ARG1(task_group_context *) ) {} +template +struct context_guard_helper { + void set_ctx( __TBB_CONTEXT_ARG1(task_group_context *) ) {} void restore_default() {} }; #endif /* __TBB_FP_CONTEXT */ diff --git a/src/tbb/src/tbb/scheduler_common.h b/src/tbb/src/tbb/scheduler_common.h index 02a39d25..19c494e6 100644 --- a/src/tbb/src/tbb/scheduler_common.h +++ b/src/tbb/src/tbb/scheduler_common.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_scheduler_common_H @@ -168,6 +164,10 @@ enum task_extra_state { es_version_1_task = 0, //! Tag for v3 tasks (i.e. tasks in TBB 2.1-2.2) es_version_3_task = 1, +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Tag for critical tasks + es_task_critical = 0x8, +#endif //! Tag for enqueued tasks es_task_enqueued = 0x10, //! Tag for v3 task_proxy. @@ -347,6 +347,11 @@ struct arena_slot_line2 { /** Modified by the owner thread (during these operations). **/ unsigned hint_for_pop; +#if __TBB_PREVIEW_CRITICAL_TASKS + //! Similar to 'hint_for_pop' but for critical tasks. + unsigned hint_for_critical; +#endif + //! Index of the element following the last ready task in the deque. /** Modified by the owner thread. **/ __TBB_atomic size_t tail; diff --git a/src/tbb/src/tbb/scheduler_utility.h b/src/tbb/src/tbb/scheduler_utility.h index 5c228598..6b6681ae 100644 --- a/src/tbb/src/tbb/scheduler_utility.h +++ b/src/tbb/src/tbb/scheduler_utility.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_scheduler_utility_H diff --git a/src/tbb/src/tbb/semaphore.cpp b/src/tbb/src/tbb/semaphore.cpp index 4c6d8532..b0e2afa0 100644 --- a/src/tbb/src/tbb/semaphore.cpp +++ b/src/tbb/src/tbb/semaphore.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "semaphore.h" diff --git a/src/tbb/src/tbb/semaphore.h b/src/tbb/src/tbb/semaphore.h index 43af24aa..472255f0 100644 --- a/src/tbb/src/tbb/semaphore.h +++ b/src/tbb/src/tbb/semaphore.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,15 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbb_semaphore_H #define __TBB_tbb_semaphore_H +#include #include "tbb/tbb_stddef.h" #if _WIN32||_WIN64 @@ -191,6 +188,8 @@ class binary_semaphore : no_copy { #if __TBB_USE_FUTEX class binary_semaphore : no_copy { +// The implementation is equivalent to the "Mutex, Take 3" one +// in the paper "Futexes Are Tricky" by Ulrich Drepper public: //! ctor binary_semaphore() { my_sem = 1; } @@ -202,7 +201,7 @@ class binary_semaphore : no_copy { if( (s = my_sem.compare_and_swap( 1, 0 ))!=0 ) { if( s!=2 ) s = my_sem.fetch_and_store( 2 ); - while( s!=0 ) { + while( s!=0 ) { // This loop deals with spurious wakeup futex_wait( &my_sem, 2 ); s = my_sem.fetch_and_store( 2 ); } @@ -211,14 +210,11 @@ class binary_semaphore : no_copy { //! post/release void V() { __TBB_ASSERT( my_sem>=1, "multiple V()'s in a row?" ); - if( my_sem--!=1 ) { - //if old value was 2 - my_sem = 0; + if( my_sem.fetch_and_store( 0 )==2 ) futex_wakeup_one( &my_sem ); - } } private: - atomic my_sem; + atomic my_sem; // 0 - open; 1 - closed, no waits; 2 - closed, possible waits }; #else typedef uint32_t sem_count_t; diff --git a/src/tbb/src/tbb/spin_mutex.cpp b/src/tbb/src/tbb/spin_mutex.cpp index 6fab769a..4a5dfde5 100644 --- a/src/tbb/src/tbb/spin_mutex.cpp +++ b/src/tbb/src/tbb/spin_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_machine.h" diff --git a/src/tbb/src/tbb/spin_rw_mutex.cpp b/src/tbb/src/tbb/spin_rw_mutex.cpp index 9b14cc1c..89a8bc7b 100644 --- a/src/tbb/src/tbb/spin_rw_mutex.cpp +++ b/src/tbb/src/tbb/spin_rw_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/spin_rw_mutex.h" diff --git a/src/tbb/src/tbb/task.cpp b/src/tbb/src/tbb/task.cpp index 12307a91..a7a83526 100644 --- a/src/tbb/src/tbb/task.cpp +++ b/src/tbb/src/tbb/task.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Do not include task.h directly. Use scheduler_common.h instead @@ -31,8 +27,6 @@ namespace tbb { -using namespace std; - namespace internal { //------------------------------------------------------------------------ diff --git a/src/tbb/src/tbb/task_group_context.cpp b/src/tbb/src/tbb/task_group_context.cpp index 4036a666..d6097251 100644 --- a/src/tbb/src/tbb/task_group_context.cpp +++ b/src/tbb/src/tbb/task_group_context.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "scheduler.h" @@ -200,6 +196,12 @@ task_group_context::~task_group_context () { } void task_group_context::init () { +#if DO_ITT_NOTIFY + // Check version of task group context to avoid reporting misleading identifier. + if( ( my_version_and_traits & version_mask ) < 3 ) + my_name = internal::CUSTOM_CTX; +#endif + ITT_TASK_GROUP(this, my_name, NULL); __TBB_STATIC_ASSERT ( sizeof(my_version_and_traits) >= 4, "Layout of my_version_and_traits must be reconsidered on this platform" ); __TBB_STATIC_ASSERT ( sizeof(task_group_context) == 2 * NFS_MaxLineSize, "Context class has wrong size - check padding and members alignment" ); __TBB_ASSERT ( (uintptr_t(this) & (sizeof(my_cancellation_requested) - 1)) == 0, "Context is improperly aligned" ); diff --git a/src/tbb/src/tbb/task_stream.h b/src/tbb/src/tbb/task_stream.h index 0118e1b8..62d357a1 100644 --- a/src/tbb/src/tbb/task_stream.h +++ b/src/tbb/src/tbb/task_stream.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_task_stream_H diff --git a/src/tbb/src/tbb/task_stream_extended.h b/src/tbb/src/tbb/task_stream_extended.h new file mode 100644 index 00000000..8103d1ef --- /dev/null +++ b/src/tbb/src/tbb/task_stream_extended.h @@ -0,0 +1,319 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_task_stream_extended_H +#define _TBB_task_stream_extended_H + +//! This file is a possible future replacement for the task_stream class implemented in +//! task_stream.h. It refactors the code and extends task_stream capabilities by moving lane +//! management during operations on caller side. Despite the fact that new implementation should not +//! affect performance of the original task stream, analysis on this subject was not made at the +//! time it was developed. In addition, it is not clearly seen at the moment that this container +//! would be suitable for critical tasks due to linear time complexity on its operations. + + +#if _TBB_task_stream_H +#error Either task_stream.h or this file can be included at the same time. +#endif + +#if !__TBB_CPF_BUILD +#error This code bears a preview status until it proves its usefulness/performance suitability. +#endif + +#include "tbb/tbb_stddef.h" +#include +#include +#include "tbb/atomic.h" // for __TBB_Atomic* +#include "tbb/spin_mutex.h" +#include "tbb/tbb_allocator.h" +#include "scheduler_common.h" +#include "tbb_misc.h" // for FastRandom + +namespace tbb { +namespace internal { + +//! Essentially, this is just a pair of a queue and a mutex to protect the queue. +/** The reason std::pair is not used is that the code would look less clean + if field names were replaced with 'first' and 'second'. **/ +template< typename T, typename mutex_t > +struct queue_and_mutex { + typedef std::deque< T, tbb_allocator > queue_base_t; + + queue_base_t my_queue; + mutex_t my_mutex; + + queue_and_mutex () : my_queue(), my_mutex() {} + ~queue_and_mutex () {} +}; + +typedef uintptr_t population_t; +const population_t one = 1; + +inline void set_one_bit( population_t& dest, int pos ) { + __TBB_ASSERT( pos>=0, NULL ); + __TBB_ASSERT( pos=0, NULL ); + __TBB_ASSERT( pos=0, NULL ); + __TBB_ASSERT( pos lane_t; +}; + +enum task_stream_accessor_type { front_accessor = 0, back_nonnull_accessor }; + +//! Specializes from which side of the underlying container elements are retrieved. Method must be +//! called under corresponding mutex locked. +template +class task_stream_accessor : public task_stream_base { +protected: + using task_stream_base::lane_t; + task* get_item( lane_t::queue_base_t& queue ) { + task* result = queue.front(); + queue.pop_front(); + return result; + } +}; + +template<> +class task_stream_accessor< back_nonnull_accessor > : public task_stream_base { +protected: + task* get_item( lane_t::queue_base_t& queue ) { + task* result = NULL; + do { + result = queue.back(); + queue.pop_back(); + } while( !result && !queue.empty() ); + return result; + } +}; + +//! The container for "fairness-oriented" aka "enqueued" tasks. +template +class task_stream : public task_stream_accessor< accessor > { + typedef typename task_stream_accessor::lane_t lane_t; + population_t population[Levels]; + padded* lanes[Levels]; + unsigned N; + +public: + task_stream() : N() { + for(int level = 0; level < Levels; level++) { + population[level] = 0; + lanes[level] = NULL; + } + } + + void initialize( unsigned n_lanes ) { + const unsigned max_lanes = sizeof(population_t) * CHAR_BIT; + + N = n_lanes>=max_lanes ? max_lanes : n_lanes>2 ? 1<<(__TBB_Log2(n_lanes-1)+1) : 2; + __TBB_ASSERT( N==max_lanes || N>=n_lanes && ((N-1)&N)==0, "number of lanes miscalculated"); + __TBB_ASSERT( N <= sizeof(population_t) * CHAR_BIT, NULL ); + for(int level = 0; level < Levels; level++) { + lanes[level] = new padded[N]; + __TBB_ASSERT( !population[level], NULL ); + } + } + + ~task_stream() { + for(int level = 0; level < Levels; level++) + if (lanes[level]) delete[] lanes[level]; + } + + //! Returns true on successful push, otherwise - false. + bool try_push( task* source, int level, unsigned lane_idx ) { + __TBB_ASSERT( 0 <= level && level < Levels, "Incorrect lane level specified." ); + spin_mutex::scoped_lock lock; + if( lock.try_acquire( lanes[level][lane_idx].my_mutex ) ) { + lanes[level][lane_idx].my_queue.push_back( source ); + set_one_bit( population[level], lane_idx ); // TODO: avoid atomic op if the bit is already set + return true; + } + return false; + } + + //! Push a task into a lane. Lane selection is performed by passed functor. + template + void push( task* source, int level, const lane_selector_t& next_lane ) { + bool succeed = false; + unsigned lane = 0; + do { + lane = next_lane( /*out_of=*/N ); + __TBB_ASSERT( lane < N, "Incorrect lane index." ); + } while( ! (succeed = try_push( source, level, lane )) ); + } + + //! Returns pointer to task on successful pop, otherwise - NULL. + task* try_pop( int level, unsigned lane_idx ) { + __TBB_ASSERT( 0 <= level && level < Levels, "Incorrect lane level specified." ); + if( !is_bit_set( population[level], lane_idx ) ) + return NULL; + task* result = NULL; + lane_t& lane = lanes[level][lane_idx]; + spin_mutex::scoped_lock lock; + if( lock.try_acquire( lane.my_mutex ) && !lane.my_queue.empty() ) { + result = this->get_item( lane.my_queue ); + if( lane.my_queue.empty() ) + clear_one_bit( population[level], lane_idx ); + } + return result; + } + + //! Try finding and popping a task using passed functor for lane selection. Last used lane is + //! updated inside lane selector. + template + task* pop( int level, const lane_selector_t& next_lane ) { + task* popped = NULL; + unsigned lane = 0; + do { + lane = next_lane( /*out_of=*/N ); + __TBB_ASSERT( lane < N, "Incorrect lane index." ); + } while( !empty( level ) && !(popped = try_pop( level, lane )) ); + return popped; + } + + // TODO: unify '*_specific' logic with 'pop' methods above + task* look_specific( __TBB_ISOLATION_ARG(task_stream_base::lane_t::queue_base_t& queue, isolation_tag isolation) ) { + __TBB_ASSERT( !queue.empty(), NULL ); + // TODO: add a worst-case performance test and consider an alternative container with better + // performance for isolation search. + typename lane_t::queue_base_t::iterator curr = queue.end(); + do { + // TODO: consider logic from get_task to simplify the code. + task* result = *--curr; + if( result __TBB_ISOLATION_EXPR( && result->prefix().isolation == isolation ) ) { + if( queue.end() - curr == 1 ) + queue.pop_back(); // a little of housekeeping along the way + else + *curr = 0; // grabbing task with the same isolation + // TODO: move one of the container's ends instead if the task has been found there + return result; + } + } while( curr != queue.begin() ); + return NULL; + } + + //! Try finding and popping a related task. + task* pop_specific( int level, __TBB_ISOLATION_ARG(unsigned& last_used_lane, isolation_tag isolation) ) { + task* result = NULL; + // Lane selection is round-robin in backward direction. + unsigned idx = last_used_lane & (N-1); + do { + if( is_bit_set( population[level], idx ) ) { + lane_t& lane = lanes[level][idx]; + spin_mutex::scoped_lock lock; + if( lock.try_acquire(lane.my_mutex) && !lane.my_queue.empty() ) { + result = look_specific( __TBB_ISOLATION_ARG(lane.my_queue, isolation) ); + if( lane.my_queue.empty() ) + clear_one_bit( population[level], idx ); + if( result ) + break; + } + } + idx=(idx-1)&(N-1); + } while( !empty(level) && idx != last_used_lane ); + last_used_lane = idx; + return result; + } + + //! Checks existence of a task. + bool empty(int level) { + return !population[level]; + } + + //! Destroys all remaining tasks in every lane. Returns the number of destroyed tasks. + /** Tasks are not executed, because it would potentially create more tasks at a late stage. + The scheduler is really expected to execute all tasks before task_stream destruction. */ + intptr_t drain() { + intptr_t result = 0; + for(int level = 0; level < Levels; level++) + for(unsigned i=0; i +#include +#include +#include + +namespace tbb { + +namespace internal { + +#if __TBB_WIN8UI_SUPPORT +static inline bool GetBoolEnvironmentVariable( const char * ) { + return false; +} + +static inline long GetIntegralEnvironmentVariable( const char * ) { + return -1; +} +#else /* __TBB_WIN8UI_SUPPORT */ +static inline bool GetBoolEnvironmentVariable( const char * name ) { + if( const char* s = std::getenv(name) ) + { + // The result is defined as true only if the environment variable contains + // no characters except one '1' character and an arbitrary number of spaces + // (including the absence of spaces). + size_t index = std::strspn(s, " "); + if (s[index] != '1') return false; + index++; + // Memory access after incrementing is safe, since the getenv() returns a + // NULL terminated string, and even if the character getting by index is '1', + // and this character is the end of string, after incrementing we will get + // an index of character, that contains '\0' + index += std::strspn(&s[index], " "); + return !s[index]; + } + return false; +} + +static inline long GetIntegralEnvironmentVariable( const char * name ) { + if( const char* s = std::getenv(name) ) + { + char* end = NULL; + errno = 0; + long value = std::strtol(s, &end, 10); + + // We have exceeded the range, value is negative or string is incovertable + if ( errno == ERANGE || value < 0 || end==s ) + { + return -1; + } + + for ( ; *end != '\0'; end++ ) + { + if ( !std::isspace(*end) ) + return -1; + } + + return value; + } + return -1; +} +#endif /* __TBB_WIN8UI_SUPPORT */ + +} // namespace internal +} // namespace tbb + +#endif // __TBB_tbb_environment_H diff --git a/src/tbb/src/tbb/tbb_main.cpp b/src/tbb/src/tbb/tbb_main.cpp index 685d28ec..18ccc51d 100644 --- a/src/tbb/src/tbb/tbb_main.cpp +++ b/src/tbb/src/tbb/tbb_main.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -64,7 +60,7 @@ bool __TBB_InitOnce::InitializationDone; #if DO_ITT_NOTIFY static bool ITT_Present; - static bool ITT_InitializationDone; + static atomic ITT_InitializationDone; #endif #if !(_WIN32||_WIN64) || __TBB_SOURCE_DIRECTLY_INCLUDED @@ -143,9 +139,7 @@ void Scheduler_OneTimeInitialization ( bool itt_present ); #if DO_ITT_NOTIFY -#if __TBB_ITT_STRUCTURE_API - -static __itt_domain *fgt_domain = NULL; +static __itt_domain *tbb_domains[ITT_NUM_DOMAINS] = {}; struct resource_string { const char *str; @@ -163,13 +157,17 @@ static resource_string strings_for_itt[] = { #undef TBB_STRING_RESOURCE static __itt_string_handle *ITT_get_string_handle(int idx) { - __TBB_ASSERT(idx >= 0, NULL); - return idx < NUM_STRINGS ? strings_for_itt[idx].itt_str_handle : NULL; + __TBB_ASSERT( idx >= 0 && idx < NUM_STRINGS, "string handle out of valid range"); + return (idx >= 0 && idx < NUM_STRINGS) ? strings_for_itt[idx].itt_str_handle : NULL; } static void ITT_init_domains() { - fgt_domain = __itt_domain_create( _T("tbb.flow") ); - fgt_domain->flags = 1; + tbb_domains[ITT_DOMAIN_MAIN] = __itt_domain_create( _T("tbb") ); + tbb_domains[ITT_DOMAIN_MAIN]->flags = 1; + tbb_domains[ITT_DOMAIN_FLOW] = __itt_domain_create( _T("tbb.flow") ); + tbb_domains[ITT_DOMAIN_FLOW]->flags = 1; + tbb_domains[ITT_DOMAIN_ALGO] = __itt_domain_create( _T("tbb.algorithm") ); + tbb_domains[ITT_DOMAIN_ALGO]->flags = 1; } static void ITT_init_strings() { @@ -187,16 +185,14 @@ static void ITT_init() { ITT_init_strings(); } -#endif // __TBB_ITT_STRUCTURE_API - /** Thread-unsafe lazy one-time initialization of tools interop. Used by both dummy handlers and general TBB one-time initialization routine. **/ void ITT_DoUnsafeOneTimeInitialization () { + // Double check ITT_InitializationDone is necessary because the first check + // in ITT_DoOneTimeInitialization is not guarded with the __TBB_InitOnce lock. if ( !ITT_InitializationDone ) { ITT_Present = (__TBB_load_ittnotify()!=0); -#if __TBB_ITT_STRUCTURE_API if (ITT_Present) ITT_init(); -#endif ITT_InitializationDone = true; ITT_SYNC_CREATE(&market::theMarketMutex, SyncType_GlobalLock, SyncObj_SchedulerInitialization); } @@ -206,9 +202,11 @@ void ITT_DoUnsafeOneTimeInitialization () { Used by dummy handlers only. **/ extern "C" void ITT_DoOneTimeInitialization() { - __TBB_InitOnce::lock(); - ITT_DoUnsafeOneTimeInitialization(); - __TBB_InitOnce::unlock(); + if ( !ITT_InitializationDone ) { + __TBB_InitOnce::lock(); + ITT_DoUnsafeOneTimeInitialization(); + __TBB_InitOnce::unlock(); + } } #endif /* DO_ITT_NOTIFY */ @@ -241,12 +239,16 @@ void DoOneTimeInitializations() { #if (_WIN32||_WIN64) && !__TBB_SOURCE_DIRECTLY_INCLUDED //! Windows "DllMain" that handles startup and shutdown of dynamic library. -extern "C" bool WINAPI DllMain( HANDLE /*hinstDLL*/, DWORD reason, LPVOID /*lpvReserved*/ ) { +extern "C" bool WINAPI DllMain( HANDLE /*hinstDLL*/, DWORD reason, LPVOID lpvReserved ) { switch( reason ) { case DLL_PROCESS_ATTACH: __TBB_InitOnce::add_ref(); break; case DLL_PROCESS_DETACH: + // Since THREAD_DETACH is not called for the main thread, call auto-termination + // here as well - but not during process shutdown (due to risk of a deadlock). + if( lpvReserved==NULL ) // library unload + governor::terminate_auto_initialized_scheduler(); __TBB_InitOnce::remove_ref(); // It is assumed that InitializationDone is not set after DLL_PROCESS_DETACH, // and thus no race on InitializationDone is possible. @@ -287,14 +289,14 @@ void call_itt_notify_v5(int t, void *ptr) { void call_itt_notify_v5(int /*t*/, void* /*ptr*/) {} #endif -#if __TBB_ITT_STRUCTURE_API - #if DO_ITT_NOTIFY - const __itt_id itt_null_id = {0, 0, 0}; static inline __itt_domain* get_itt_domain( itt_domain_enum idx ) { - return ( idx == ITT_DOMAIN_FLOW ) ? fgt_domain : NULL; + if (tbb_domains[idx] == NULL) { + ITT_DoOneTimeInitialization(); + } + return tbb_domains[idx]; } static inline void itt_id_make(__itt_id *id, void* addr, unsigned long long extra) { @@ -351,7 +353,9 @@ void itt_task_begin_v7( itt_domain_enum domain, void *task, unsigned long long t if ( __itt_domain *d = get_itt_domain( domain ) ) { __itt_id task_id = itt_null_id; __itt_id parent_id = itt_null_id; - itt_id_make( &task_id, task, task_extra ); + if ( task ) { + itt_id_make( &task_id, task, task_extra ); + } if ( parent ) { itt_id_make( &parent_id, parent, parent_extra ); } @@ -389,29 +393,27 @@ void itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long #else // DO_ITT_NOTIFY -void itt_make_task_group_v7( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { } +void itt_make_task_group_v7( itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) { } -void itt_metadata_str_add_v7( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { } +void itt_metadata_str_add_v7( itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, + string_index /*key*/, const char* /*value*/ ) { } -void itt_relation_add_v7( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { } +void itt_relation_add_v7( itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/, + itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) { } -void itt_task_begin_v7( itt_domain_enum domain, void *task, unsigned long long task_extra, - void * /*parent*/, unsigned long long /* parent_extra */, string_index /* name_index */ ) { } +void itt_task_begin_v7( itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) { } -void itt_task_end_v7( itt_domain_enum domain ) { } +void itt_task_end_v7( itt_domain_enum /*domain*/ ) { } -void itt_region_begin_v9( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index /* name_index */ ) { } +void itt_region_begin_v9( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/, + void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) { } -void itt_region_end_v9( itt_domain_enum domain, void *region, unsigned long long region_extra ) { } +void itt_region_end_v9( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) { } #endif // DO_ITT_NOTIFY -#endif // __TBB_ITT_STRUCTURE_API - void* itt_load_pointer_v3( const void* src ) { //TODO: replace this with __TBB_load_relaxed void* result = *static_cast(src); @@ -476,8 +478,8 @@ class stack_size_control : public padded { return tbb::internal::ThreadStackSize; } virtual void apply_active() const __TBB_override { -#if __TBB_WIN8UI_SUPPORT - __TBB_ASSERT( false, "For Windows Store* apps we must not set stack size" ); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + __TBB_ASSERT( false, "For Windows 8 Store* apps we must not set stack size" ); #endif } }; diff --git a/src/tbb/src/tbb/tbb_main.h b/src/tbb/src/tbb/tbb_main.h index 24d77d11..babca1a8 100644 --- a/src/tbb/src/tbb/tbb_main.h +++ b/src/tbb/src/tbb/tbb_main.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,16 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tbb_main_H #define _TBB_tbb_main_H #include "tbb/atomic.h" +#include "governor.h" +#include "tbb_environment.h" namespace tbb { @@ -73,6 +71,7 @@ class __TBB_InitOnce { //! Remove the initial reference to resources. /** This is not necessarily the last reference if other threads are still running. **/ ~__TBB_InitOnce() { + governor::terminate_auto_initialized_scheduler(); // TLS dtor not called for the main thread remove_ref(); // We assume that InitializationDone is not set after file-scope destructors // start running, and thus no race on InitializationDone is possible. diff --git a/src/tbb/src/tbb/tbb_misc.cpp b/src/tbb/src/tbb/tbb_misc.cpp index 5e389fec..1e71cdc9 100644 --- a/src/tbb/src/tbb/tbb_misc.cpp +++ b/src/tbb/src/tbb/tbb_misc.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Source file for miscellaneous entities that are infrequently referenced by @@ -46,8 +42,6 @@ #include #endif -using namespace std; - namespace tbb { const char* bad_last_alloc::what() const throw() { return "bad allocation in previous or concurrent attempt"; } @@ -65,7 +59,7 @@ namespace internal { fprintf (stderr, "Exception %s with message %s would've been thrown, " \ "if exception handling were not disabled. Aborting.\n", exc_name, msg); \ fflush(stderr); \ - abort(); + std::abort(); #define DO_THROW(exc, init_args) PRINT_ERROR_AND_ABORT(#exc, #init_args) #endif /* !TBB_USE_EXCEPTIONS */ @@ -88,7 +82,7 @@ void handle_perror( int error_code, const char* what ) { // Ensure that buffer ends in terminator. buf[sizeof(buf)-1] = 0; #if TBB_USE_EXCEPTIONS - throw runtime_error(buf); + throw std::runtime_error(buf); #else PRINT_ERROR_AND_ABORT( "runtime_error", buf); #endif /* !TBB_USE_EXCEPTIONS */ @@ -105,7 +99,7 @@ void handle_win_error( int error_code ) { sprintf_s((char*)&buf, 512, "error code %d", error_code); #endif #if TBB_USE_EXCEPTIONS - throw runtime_error(buf); + throw std::runtime_error(buf); #else PRINT_ERROR_AND_ABORT( "runtime_error", buf); #endif /* !TBB_USE_EXCEPTIONS */ @@ -119,27 +113,27 @@ void throw_bad_last_alloc_exception_v4() { void throw_exception_v4 ( exception_id eid ) { __TBB_ASSERT ( eid > 0 && eid < eid_max, "Unknown exception ID" ); switch ( eid ) { - case eid_bad_alloc: DO_THROW( bad_alloc, () ); + case eid_bad_alloc: DO_THROW(std::bad_alloc, () ); case eid_bad_last_alloc: DO_THROW( bad_last_alloc, () ); - case eid_nonpositive_step: DO_THROW( invalid_argument, ("Step must be positive") ); - case eid_out_of_range: DO_THROW( out_of_range, ("Index out of requested size range") ); - case eid_segment_range_error: DO_THROW( range_error, ("Index out of allocated segment slots") ); - case eid_index_range_error: DO_THROW( range_error, ("Index is not allocated") ); + case eid_nonpositive_step: DO_THROW(std::invalid_argument, ("Step must be positive") ); + case eid_out_of_range: DO_THROW(std::out_of_range, ("Index out of requested size range") ); + case eid_segment_range_error: DO_THROW(std::range_error, ("Index out of allocated segment slots") ); + case eid_index_range_error: DO_THROW(std::range_error, ("Index is not allocated") ); case eid_missing_wait: DO_THROW( missing_wait, () ); case eid_invalid_multiple_scheduling: DO_THROW( invalid_multiple_scheduling, () ); case eid_improper_lock: DO_THROW( improper_lock, () ); - case eid_possible_deadlock: DO_THROW( runtime_error, ("Resource deadlock would occur") ); - case eid_operation_not_permitted: DO_THROW( runtime_error, ("Operation not permitted") ); - case eid_condvar_wait_failed: DO_THROW( runtime_error, ("Wait on condition variable failed") ); - case eid_invalid_load_factor: DO_THROW( out_of_range, ("Invalid hash load factor") ); - case eid_reserved: DO_THROW( out_of_range, ("[backward compatibility] Invalid number of buckets") ); - case eid_invalid_swap: DO_THROW( invalid_argument, ("swap() is invalid on non-equal allocators") ); - case eid_reservation_length_error: DO_THROW( length_error, ("reservation size exceeds permitted max size") ); - case eid_invalid_key: DO_THROW( out_of_range, ("invalid key") ); + case eid_possible_deadlock: DO_THROW(std::runtime_error, ("Resource deadlock would occur") ); + case eid_operation_not_permitted: DO_THROW(std::runtime_error, ("Operation not permitted") ); + case eid_condvar_wait_failed: DO_THROW(std::runtime_error, ("Wait on condition variable failed") ); + case eid_invalid_load_factor: DO_THROW(std::out_of_range, ("Invalid hash load factor") ); + case eid_reserved: DO_THROW(std::out_of_range, ("[backward compatibility] Invalid number of buckets") ); + case eid_invalid_swap: DO_THROW(std::invalid_argument, ("swap() is invalid on non-equal allocators") ); + case eid_reservation_length_error: DO_THROW(std::length_error, ("reservation size exceeds permitted max size") ); + case eid_invalid_key: DO_THROW(std::out_of_range, ("invalid key") ); case eid_user_abort: DO_THROW( user_abort, () ); - case eid_bad_tagged_msg_cast: DO_THROW( runtime_error, ("Illegal tagged_msg cast") ); + case eid_bad_tagged_msg_cast: DO_THROW(std::runtime_error, ("Illegal tagged_msg cast") ); #if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - case eid_blocking_thread_join_impossible: DO_THROW( runtime_error, ("Blocking terminate failed") ); + case eid_blocking_thread_join_impossible: DO_THROW(std::runtime_error, ("Blocking terminate failed") ); #endif default: break; } @@ -191,16 +185,6 @@ void fix_broken_rethrow() {} bool gcc_rethrow_exception_broken() { return false; } #endif /* __TBB_STD_RETHROW_EXCEPTION_POSSIBLY_BROKEN */ -#if __TBB_WIN8UI_SUPPORT -bool GetBoolEnvironmentVariable( const char * ) { return false;} -#else /* __TBB_WIN8UI_SUPPORT */ -bool GetBoolEnvironmentVariable( const char * name ) { - if( const char* s = getenv(name) ) - return strcmp(s,"0") != 0; - return false; -} -#endif /* __TBB_WIN8UI_SUPPORT */ - /** The leading "\0" is here so that applying "strings" to the binary delivers a clean result. */ static const char VersionString[] = "\0" TBB_VERSION_STRINGS; diff --git a/src/tbb/src/tbb/tbb_misc.h b/src/tbb/src/tbb/tbb_misc.h index 8a05197e..6042bb5e 100644 --- a/src/tbb/src/tbb/tbb_misc.h +++ b/src/tbb/src/tbb/tbb_misc.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tbb_misc_H @@ -43,8 +39,9 @@ namespace internal { const size_t MByte = 1024*1024; -#if __TBB_WIN8UI_SUPPORT -// In Win8UI mode, TBB uses a thread creation API that does not allow to specify the stack size. +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) +// In Win8UI mode (Windows 8 Store* applications), TBB uses a thread creation API +// that does not allow to specify the stack size. // Still, the thread stack size value, either explicit or default, is used by the scheduler. // So here we set the default value to match the platform's default of 1MB. const size_t ThreadStackSize = 1*MByte; @@ -83,9 +80,6 @@ void MoveThreadIntoProcessorGroup( void* hThread, int groupIndex ); //! Throws std::runtime_error with what() returning error_code description prefixed with aux_info void handle_win_error( int error_code ); -//! True if environment variable with given name is set and not 0; otherwise false. -bool GetBoolEnvironmentVariable( const char * name ); - //! Prints TBB version information on stderr void PrintVersion(); diff --git a/src/tbb/src/tbb/tbb_misc_ex.cpp b/src/tbb/src/tbb/tbb_misc_ex.cpp index 264abe6a..bfa5830d 100644 --- a/src/tbb/src/tbb/tbb_misc_ex.cpp +++ b/src/tbb/src/tbb/tbb_misc_ex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Source file for miscellaneous entities that are infrequently referenced by @@ -61,7 +57,7 @@ namespace internal { static int (*libiomp_try_restoring_original_mask)(); // Table for mapping to libiomp entry points static const dynamic_link_descriptor iompLinkTable[] = { - { "kmp_set_thread_affinity_mask_initial", (pointer_to_handler*)(void*)(&libiomp_try_restoring_original_mask) } + DLD_NOWEAK( kmp_set_thread_affinity_mask_initial, libiomp_try_restoring_original_mask ) }; #endif diff --git a/src/tbb/src/tbb/tbb_resource.rc b/src/tbb/src/tbb/tbb_resource.rc index 7dc4fe6f..19822d90 100644 --- a/src/tbb/src/tbb/tbb_resource.rc +++ b/src/tbb/src/tbb/tbb_resource.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // Microsoft Visual C++ generated resource script. // @@ -81,7 +77,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Intel(R) Threading Building Blocks library\0" VALUE "FileVersion", TBB_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalCopyright", "Copyright 2005-2019 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG VALUE "OriginalFilename", "tbb.dll\0" diff --git a/src/tbb/src/tbb/tbb_statistics.cpp b/src/tbb/src/tbb/tbb_statistics.cpp index 3656f9ab..2ca03b37 100644 --- a/src/tbb/src/tbb/tbb_statistics.cpp +++ b/src/tbb/src/tbb/tbb_statistics.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb_statistics.h" diff --git a/src/tbb/src/tbb/tbb_statistics.h b/src/tbb/src/tbb/tbb_statistics.h index 390aa36a..fb5d9ccd 100644 --- a/src/tbb/src/tbb/tbb_statistics.h +++ b/src/tbb/src/tbb/tbb_statistics.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tbb_statistics_H @@ -208,7 +204,7 @@ struct statistics_counters { statistics_counters() { reset(); } - void reset () { memset( (void*) this, 0, sizeof(statistics_counters) ); } + void reset () { memset( this, 0, sizeof(statistics_counters) ); } counter_type& field ( size_t index ) { return reinterpret_cast(this)[index]; } diff --git a/src/tbb/src/tbb/tbb_thread.cpp b/src/tbb/src/tbb/tbb_thread.cpp index f64fb1f3..7de94922 100644 --- a/src/tbb/src/tbb/tbb_thread.cpp +++ b/src/tbb/src/tbb/tbb_thread.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #if _WIN32||_WIN64 diff --git a/src/tbb/src/tbb/tbb_version.h b/src/tbb/src/tbb/tbb_version.h index 89a1c516..f8606f26 100644 --- a/src/tbb/src/tbb/tbb_version.h +++ b/src/tbb/src/tbb/tbb_version.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ // Please define version number in the file: diff --git a/src/tbb/src/tbb/tls.h b/src/tbb/src/tbb/tls.h index fd4355e8..ba6f2dde 100644 --- a/src/tbb/src/tbb/tls.h +++ b/src/tbb/src/tbb/tls.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_tls_H diff --git a/src/tbb/src/tbb/tools_api/disable_warnings.h b/src/tbb/src/tbb/tools_api/disable_warnings.h index a2452bb6..72eda850 100644 --- a/src/tbb/src/tbb/tools_api/disable_warnings.h +++ b/src/tbb/src/tbb/tools_api/disable_warnings.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "ittnotify_config.h" diff --git a/src/tbb/src/tbb/tools_api/ittnotify.h b/src/tbb/src/tbb/tools_api/ittnotify.h index 037588ce..b6fbef5f 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify.h +++ b/src/tbb/src/tbb/tools_api/ittnotify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_H_ @@ -2709,7 +2705,7 @@ ITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info /** * @ingroup clockdomains - * @brief Recalculate clock domains frequences and clock base timestamps. + * @brief Recalculate clock domains frequencies and clock base timestamps. */ void ITTAPI __itt_clock_domain_reset(void); @@ -3719,7 +3715,7 @@ ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void)) /** @endcond */ /** - * @brief Destroy the inforamtion about stitch point identified by the pointer previously returned by __itt_stack_caller_create() + * @brief Destroy the information about stitch point identified by the pointer previously returned by __itt_stack_caller_create() */ void ITTAPI __itt_stack_caller_destroy(__itt_caller id); diff --git a/src/tbb/src/tbb/tools_api/ittnotify_config.h b/src/tbb/src/tbb/tools_api/ittnotify_config.h index 1f90963b..84af62df 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_config.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_config.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_CONFIG_H_ @@ -294,7 +290,16 @@ ITT_INLINE long __itt_interlocked_increment(volatile long* ptr) #ifdef SDL_STRNCPY_S #define __itt_fstrcpyn(s1, b, s2, l) SDL_STRNCPY_S(s1, b, s2, l) #else -#define __itt_fstrcpyn(s1, b, s2, l) memcpy(s1, s2, l) +#define __itt_fstrcpyn(s1, b, s2, l) { \ + if (b > 0) { \ + /* 'volatile' is used to suppress the warning that a destination */ \ + /* bound depends on the length of the source. */ \ + volatile size_t num_to_copy = (size_t)(b - 1) < (size_t)(l) ? \ + (size_t)(b - 1) : (size_t)(l); \ + strncpy(s1, s2, num_to_copy); \ + s1[num_to_copy] = 0; \ + } \ +} #endif /* SDL_STRNCPY_S */ #define __itt_fstrdup(s) strdup(s) diff --git a/src/tbb/src/tbb/tools_api/ittnotify_static.c b/src/tbb/src/tbb/tools_api/ittnotify_static.c index c0867459..08c69b36 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_static.c +++ b/src/tbb/src/tbb/tools_api/ittnotify_static.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "ittnotify_config.h" @@ -32,7 +28,7 @@ #include #include -#define INTEL_NO_MACRO_BODY +#define INTEL_NO_MACRO_BODY #define INTEL_ITTNOTIFY_API_PRIVATE #include "ittnotify.h" #include "legacy/ittnotify.h" @@ -378,7 +374,7 @@ static __itt_string_handle* ITTAPI ITT_VERSIONIZE(ITT_JOIN(_N_(string_handle_cre { if (h->strW != NULL && !wcscmp(h->strW, name)) break; } - if (h == NULL) + if (h == NULL) { NEW_STRING_HANDLE_W(&_N_(_ittapi_global),h,h_tail,name); } @@ -607,7 +603,7 @@ static const char* __itt_fsplit(const char* s, const char* sep, const char** out /* This function return value of env variable that placed into static buffer. * !!! The same static buffer is used for subsequent calls. !!! - * This was done to aviod dynamic allocation for few calls. + * This was done to avoid dynamic allocation for few calls. * Actually we need this function only four times. */ static const char* __itt_get_env_var(const char* name) @@ -940,6 +936,7 @@ ITT_EXTERN_C int _N_(init_ittlib)(const char* lib_name, __itt_group_id init_grou switch (lib_version) { case 0: groups = __itt_group_legacy; + /* Falls through */ case 1: /* Fill all pointers from dynamic library */ for (i = 0; _N_(_ittapi_global).api_list_ptr[i].name != NULL; i++) diff --git a/src/tbb/src/tbb/tools_api/ittnotify_static.h b/src/tbb/src/tbb/tools_api/ittnotify_static.h index 6907b835..58c1a4bb 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_static.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_static.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "ittnotify_config.h" diff --git a/src/tbb/src/tbb/tools_api/ittnotify_types.h b/src/tbb/src/tbb/tools_api/ittnotify_types.h index 83cb0ddf..acca4e87 100644 --- a/src/tbb/src/tbb/tools_api/ittnotify_types.h +++ b/src/tbb/src/tbb/tools_api/ittnotify_types.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _ITTNOTIFY_TYPES_H_ diff --git a/src/tbb/src/tbb/tools_api/legacy/ittnotify.h b/src/tbb/src/tbb/tools_api/legacy/ittnotify.h index e15734a4..17056e05 100644 --- a/src/tbb/src/tbb/tools_api/legacy/ittnotify.h +++ b/src/tbb/src/tbb/tools_api/legacy/ittnotify.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _LEGACY_ITTNOTIFY_H_ diff --git a/src/tbb/src/tbb/win32-tbb-export.def b/src/tbb/src/tbb/win32-tbb-export.def index 9f1b7dca..82c2ea2e 100644 --- a/src/tbb/src/tbb/win32-tbb-export.def +++ b/src/tbb/src/tbb/win32-tbb-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS diff --git a/src/tbb/src/tbb/win32-tbb-export.lst b/src/tbb/src/tbb/win32-tbb-export.lst index 35b1833a..a6c4b786 100644 --- a/src/tbb/src/tbb/win32-tbb-export.lst +++ b/src/tbb/src/tbb/win32-tbb-export.lst @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; #include "tbb/tbb_config.h" @@ -138,7 +134,6 @@ __TBB_SYMBOL( ?itt_store_pointer_with_release_v3@internal@tbb@@YAXPAX0@Z ) __TBB_SYMBOL( ?call_itt_notify_v5@internal@tbb@@YAXHPAX@Z ) __TBB_SYMBOL( ?itt_set_sync_name_v3@internal@tbb@@YAXPAXPB_W@Z ) __TBB_SYMBOL( ?itt_load_pointer_v3@internal@tbb@@YAPAXPBX@Z ) -#if __TBB_ITT_STRUCTURE_API __TBB_SYMBOL( ?itt_make_task_group_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4string_index@12@@Z ) __TBB_SYMBOL( ?itt_metadata_str_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_KW4string_index@12@PBD@Z ) __TBB_SYMBOL( ?itt_relation_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_KW4itt_relation@12@12@Z ) @@ -146,7 +141,6 @@ __TBB_SYMBOL( ?itt_task_begin_v7@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4 __TBB_SYMBOL( ?itt_task_end_v7@internal@tbb@@YAXW4itt_domain_enum@12@@Z ) __TBB_SYMBOL( ?itt_region_begin_v9@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K12W4string_index@12@@Z ) __TBB_SYMBOL( ?itt_region_end_v9@internal@tbb@@YAXW4itt_domain_enum@12@PAX_K@Z ) -#endif // pipeline.cpp __TBB_SYMBOL( ??0pipeline@tbb@@QAE@XZ ) diff --git a/src/tbb/src/tbb/win64-gcc-tbb-export.def b/src/tbb/src/tbb/win64-gcc-tbb-export.def index 6159cf85..944aae7c 100644 --- a/src/tbb/src/tbb/win64-gcc-tbb-export.def +++ b/src/tbb/src/tbb/win64-gcc-tbb-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbb/win64-gcc-tbb-export.lst b/src/tbb/src/tbb/win64-gcc-tbb-export.lst index 46cb2985..452c137f 100644 --- a/src/tbb/src/tbb/win64-gcc-tbb-export.lst +++ b/src/tbb/src/tbb/win64-gcc-tbb-export.lst @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -166,6 +162,14 @@ __TBB_SYMBOL( _ZN3tbb8internal33itt_store_pointer_with_release_v3EPvS1_ ) __TBB_SYMBOL( _ZN3tbb8internal18call_itt_notify_v5EiPv ) __TBB_SYMBOL( _ZN3tbb8internal20itt_set_sync_name_v3EPvPKc ) __TBB_SYMBOL( _ZN3tbb8internal19itt_load_pointer_v3EPKv ) +__TBB_SYMBOL( _ZN3tbb8internal22itt_make_task_group_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal23itt_metadata_str_add_v7ENS0_15itt_domain_enumEPvyNS0_12string_indexEPKc ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_relation_add_v7ENS0_15itt_domain_enumEPvyNS0_12itt_relationES2_y ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_task_begin_v7ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal15itt_task_end_v7ENS0_15itt_domain_enumE ) +__TBB_SYMBOL( _ZN3tbb8internal19itt_region_begin_v9ENS0_15itt_domain_enumEPvyS2_yNS0_12string_indexE ) +__TBB_SYMBOL( _ZN3tbb8internal17itt_region_end_v9ENS0_15itt_domain_enumEPvy ) + /* pipeline.cpp */ __TBB_SYMBOL( _ZTIN3tbb6filterE ) diff --git a/src/tbb/src/tbb/win64-tbb-export.def b/src/tbb/src/tbb/win64-tbb-export.def index d3f3d1a5..e4c22e7b 100644 --- a/src/tbb/src/tbb/win64-tbb-export.def +++ b/src/tbb/src/tbb/win64-tbb-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; ; This file is organized with a section for each .cpp file. ; Each of these sections is in alphabetical order. diff --git a/src/tbb/src/tbb/win64-tbb-export.lst b/src/tbb/src/tbb/win64-tbb-export.lst index 3bca01a9..b044bbe9 100644 --- a/src/tbb/src/tbb/win64-tbb-export.lst +++ b/src/tbb/src/tbb/win64-tbb-export.lst @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; // This file is organized with a section for each .cpp file. // Each of these sections is in alphabetical order. @@ -133,7 +129,6 @@ __TBB_SYMBOL( ?itt_store_pointer_with_release_v3@internal@tbb@@YAXPEAX0@Z ) __TBB_SYMBOL( ?call_itt_notify_v5@internal@tbb@@YAXHPEAX@Z ) __TBB_SYMBOL( ?itt_load_pointer_v3@internal@tbb@@YAPEAXPEBX@Z ) __TBB_SYMBOL( ?itt_set_sync_name_v3@internal@tbb@@YAXPEAXPEB_W@Z ) -#if __TBB_ITT_STRUCTURE_API __TBB_SYMBOL( ?itt_make_task_group_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W4string_index@12@@Z ) __TBB_SYMBOL( ?itt_metadata_str_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_KW4string_index@12@PEBD@Z ) __TBB_SYMBOL( ?itt_relation_add_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_KW4itt_relation@12@12@Z ) @@ -141,7 +136,6 @@ __TBB_SYMBOL( ?itt_task_begin_v7@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W __TBB_SYMBOL( ?itt_task_end_v7@internal@tbb@@YAXW4itt_domain_enum@12@@Z ) __TBB_SYMBOL( ?itt_region_begin_v9@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K12W4string_index@12@@Z ) __TBB_SYMBOL( ?itt_region_end_v9@internal@tbb@@YAXW4itt_domain_enum@12@PEAX_K@Z ) -#endif // pipeline.cpp __TBB_SYMBOL( ??_7pipeline@tbb@@6B@ ) diff --git a/src/tbb/src/tbb/winrt-tbb-export.lst b/src/tbb/src/tbb/winrt-tbb-export.lst index da551170..1fccb4f8 100644 --- a/src/tbb/src/tbb/winrt-tbb-export.lst +++ b/src/tbb/src/tbb/winrt-tbb-export.lst @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; #include "tbb/tbb_config.h" diff --git a/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp b/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp index 7f2027fa..e5218307 100644 --- a/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp +++ b/src/tbb/src/tbb/x86_rtm_rw_mutex.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -80,7 +76,7 @@ void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) { } #endif __TBB_machine_end_transaction(); - s.my_scoped_lock.internal_set_mutex(NULL); + s.my_scoped_lock.mutex = NULL; } break; case RTM_real_reader: @@ -94,6 +90,7 @@ void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) { break; case RTM_not_in_mutex: __TBB_ASSERT(false, "RTM_not_in_mutex, but in release"); + break; default: __TBB_ASSERT(false, "invalid transaction_state"); } @@ -126,7 +123,9 @@ void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, } #endif s.transaction_state = RTM_transacting_writer; - s.my_scoped_lock.internal_set_mutex(this); // need mutex for release() + // Don not wrap the following assignment to a function, + // because it can abort the transaction in debug. Need mutex for release(). + s.my_scoped_lock.mutex = this; return; // successfully started speculation } ++num_retries; @@ -169,7 +168,9 @@ void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, } #endif s.transaction_state = RTM_transacting_reader; - s.my_scoped_lock.internal_set_mutex(this); // need mutex for release() + // Don not wrap the following assignment to a function, + // because it can abort the transaction in debug. Need mutex for release(). + s.my_scoped_lock.mutex = this; return; // successfully started speculation } // fallback path diff --git a/src/tbb/src/tbbmalloc/Customize.h b/src/tbb/src/tbbmalloc/Customize.h index 4fdf0689..f5497bc5 100644 --- a/src/tbb/src/tbbmalloc/Customize.h +++ b/src/tbb/src/tbbmalloc/Customize.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _TBB_malloc_Customize_H_ @@ -24,12 +20,13 @@ // customizing MALLOC_ASSERT macro #include "tbb/tbb_stddef.h" #define MALLOC_ASSERT(assertion, message) __TBB_ASSERT(assertion, message) +#define MALLOC_ASSERT_EX(assertion, message) __TBB_ASSERT_EX(assertion, message) #ifndef MALLOC_DEBUG #define MALLOC_DEBUG TBB_USE_DEBUG #endif -#include "tbb/tbb_machine.h" +#include "Synchronize.h" #if DO_ITT_NOTIFY #include "tbb/itt_notify.h" @@ -46,88 +43,6 @@ #define MALLOC_ITT_FINI_ITTLIB() ((void)0) #endif -//! Stripped down version of spin_mutex. -/** Instances of MallocMutex must be declared in memory that is zero-initialized. - There are no constructors. This is a feature that lets it be - used in situations where the mutex might be used while file-scope constructors - are running. - - There are no methods "acquire" or "release". The scoped_lock must be used - in a strict block-scoped locking pattern. Omitting these methods permitted - further simplification. */ -class MallocMutex : tbb::internal::no_copy { - __TBB_atomic_flag flag; - -public: - class scoped_lock : tbb::internal::no_copy { - MallocMutex& mutex; - bool taken; - public: - scoped_lock( MallocMutex& m ) : mutex(m), taken(true) { __TBB_LockByte(m.flag); } - scoped_lock( MallocMutex& m, bool block, bool *locked ) : mutex(m), taken(false) { - if (block) { - __TBB_LockByte(m.flag); - taken = true; - } else { - taken = __TBB_TryLockByte(m.flag); - } - if (locked) *locked = taken; - } - ~scoped_lock() { - if (taken) __TBB_UnlockByte(mutex.flag); - } - }; - friend class scoped_lock; -}; - -// TODO: use signed/unsigned in atomics more consistently -inline intptr_t AtomicIncrement( volatile intptr_t& counter ) { - return __TBB_FetchAndAddW( &counter, 1 )+1; -} - -inline uintptr_t AtomicAdd( volatile intptr_t& counter, intptr_t value ) { - return __TBB_FetchAndAddW( &counter, value ); -} - -inline intptr_t AtomicCompareExchange( volatile intptr_t& location, intptr_t new_value, intptr_t comparand) { - return __TBB_CompareAndSwapW( &location, new_value, comparand ); -} - -inline uintptr_t AtomicFetchStore(volatile void* location, uintptr_t value) { - return __TBB_FetchAndStoreW(location, value); -} - -inline void AtomicOr(volatile void *operand, uintptr_t addend) { - __TBB_AtomicOR(operand, addend); -} - -inline void AtomicAnd(volatile void *operand, uintptr_t addend) { - __TBB_AtomicAND(operand, addend); -} - -inline intptr_t FencedLoad( const volatile intptr_t &location ) { - return __TBB_load_with_acquire(location); -} - -inline void FencedStore( volatile intptr_t &location, intptr_t value ) { - __TBB_store_with_release(location, value); -} - -inline void SpinWaitWhileEq(const volatile intptr_t &location, const intptr_t value) { - tbb::internal::spin_wait_while_eq(location, value); -} - -class AtomicBackoff { - tbb::internal::atomic_backoff backoff; -public: - AtomicBackoff() {} - void pause() { backoff.pause(); } -}; - -inline void SpinWaitUntilEq(const volatile intptr_t &location, const intptr_t value) { - tbb::internal::spin_wait_until_eq(location, value); -} - inline intptr_t BitScanRev(uintptr_t x) { return !x? -1 : __TBB_Log2(x); } diff --git a/src/tbb/src/tbbmalloc/MapMemory.h b/src/tbb/src/tbbmalloc/MapMemory.h index 552917a4..1b63ee60 100644 --- a/src/tbb/src/tbbmalloc/MapMemory.h +++ b/src/tbb/src/tbbmalloc/MapMemory.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _itt_shared_malloc_MapMemory_H @@ -57,20 +53,108 @@ void *ErrnoPreservingMalloc(size_t bytes) #undef XPG4_WAS_DEFINED #endif -#define MEMORY_MAPPING_USES_MALLOC 0 -void* MapMemory (size_t bytes, bool hugePages) -{ - void* result = 0; - int prevErrno = errno; +inline void* mmap_impl(size_t map_size, void* map_hint = NULL, int map_flags = 0) { #ifndef MAP_ANONYMOUS // macOS* defines MAP_ANON, which is deprecated in Linux*. #define MAP_ANONYMOUS MAP_ANON #endif /* MAP_ANONYMOUS */ - int addFlags = hugePages? __TBB_MAP_HUGETLB : 0; - result = mmap(NULL, bytes, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|addFlags, -1, 0); - if (result==MAP_FAILED) + return mmap(map_hint, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | map_flags, -1, 0); +} + +inline void* mmapTHP(size_t bytes) { + // Initializes in zero-initialized data section + static void* hint; + + // Optimistically try to use a last huge page aligned region end + // as a hint for mmap. + hint = hint ? (void*)((uintptr_t)hint - bytes) : hint; + void* result = mmap_impl(bytes, hint); + + // Something went wrong + if (result == MAP_FAILED) { + hint = NULL; + return MAP_FAILED; + } + + // Otherwise, fall back to the slow path - map oversized region + // and trim excess parts. + if (!isAligned(result, HUGE_PAGE_SIZE)) { + // Undo previous try + munmap(result, bytes); + + // Map oversized on huge page size region + result = mmap_impl(bytes + HUGE_PAGE_SIZE); + + // Something went wrong + if (result == MAP_FAILED) { + hint = NULL; + return MAP_FAILED; + } + + // Misalignment offset + uintptr_t offset = 0; + + if (!isAligned(result, HUGE_PAGE_SIZE)) { + // Trim excess head of a region if it is no aligned + offset = HUGE_PAGE_SIZE - ((uintptr_t)result & (HUGE_PAGE_SIZE - 1)); + munmap(result, offset); + + // New region beginning + result = (void*)((uintptr_t)result + offset); + } + + // Trim excess tail of a region + munmap((void*)((uintptr_t)result + bytes), HUGE_PAGE_SIZE - offset); + } + + // Assume, that mmap virtual addresses grow down by default + // So, set a hint as a result of a last successful allocation + // and then use it minus requested size as a new mapping point. + // TODO: Atomic store is meant here, fence not needed, but + // currently we don't have such function. + hint = result; + + MALLOC_ASSERT(isAligned(result, HUGE_PAGE_SIZE), "Mapped address is not aligned on huge page size."); + + return result; +} + +#define MEMORY_MAPPING_USES_MALLOC 0 +void* MapMemory (size_t bytes, PageType pageType) +{ + void* result = 0; + int prevErrno = errno; + + switch (pageType) { + case REGULAR: + { + result = mmap_impl(bytes); + break; + } + case PREALLOCATED_HUGE_PAGE: + { + MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, "Mapping size should be divisible by huge page size"); + result = mmap_impl(bytes, NULL, __TBB_MAP_HUGETLB); + break; + } + case TRANSPARENT_HUGE_PAGE: + { + MALLOC_ASSERT((bytes % HUGE_PAGE_SIZE) == 0, "Mapping size should be divisible by huge page size"); + result = mmapTHP(bytes); + break; + } + default: + { + MALLOC_ASSERT(false, "Unknown page type"); + } + } + + if (result == MAP_FAILED) { errno = prevErrno; - return result==MAP_FAILED? 0: result; + return 0; + } + + return result; } int UnmapMemory(void *area, size_t bytes) @@ -86,7 +170,7 @@ int UnmapMemory(void *area, size_t bytes) #include #define MEMORY_MAPPING_USES_MALLOC 0 -void* MapMemory (size_t bytes, bool) +void* MapMemory (size_t bytes, PageType) { /* Is VirtualAlloc thread safe? */ return VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); @@ -101,7 +185,7 @@ int UnmapMemory(void *area, size_t /*bytes*/) #else #define MEMORY_MAPPING_USES_MALLOC 1 -void* MapMemory (size_t bytes, bool) +void* MapMemory (size_t bytes, PageType) { return ErrnoPreservingMalloc( bytes ); } diff --git a/src/tbb/src/tbbmalloc/Statistics.h b/src/tbb/src/tbbmalloc/Statistics.h index 392442e9..ad197317 100644 --- a/src/tbb/src/tbbmalloc/Statistics.h +++ b/src/tbb/src/tbbmalloc/Statistics.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #define MAX_THREADS 1024 diff --git a/src/tbb/src/tbbmalloc/Synchronize.h b/src/tbb/src/tbbmalloc/Synchronize.h new file mode 100644 index 00000000..f38f69d2 --- /dev/null +++ b/src/tbb/src/tbbmalloc/Synchronize.h @@ -0,0 +1,104 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_malloc_Synchronize_H_ +#define __TBB_malloc_Synchronize_H_ + +#include "tbb/tbb_machine.h" + +//! Stripped down version of spin_mutex. +/** Instances of MallocMutex must be declared in memory that is zero-initialized. + There are no constructors. This is a feature that lets it be + used in situations where the mutex might be used while file-scope constructors + are running. + + There are no methods "acquire" or "release". The scoped_lock must be used + in a strict block-scoped locking pattern. Omitting these methods permitted + further simplification. */ +class MallocMutex : tbb::internal::no_copy { + __TBB_atomic_flag flag; + +public: + class scoped_lock : tbb::internal::no_copy { + MallocMutex& mutex; + bool taken; + public: + scoped_lock( MallocMutex& m ) : mutex(m), taken(true) { __TBB_LockByte(m.flag); } + scoped_lock( MallocMutex& m, bool block, bool *locked ) : mutex(m), taken(false) { + if (block) { + __TBB_LockByte(m.flag); + taken = true; + } else { + taken = __TBB_TryLockByte(m.flag); + } + if (locked) *locked = taken; + } + ~scoped_lock() { + if (taken) __TBB_UnlockByte(mutex.flag); + } + }; + friend class scoped_lock; +}; + +// TODO: use signed/unsigned in atomics more consistently +inline intptr_t AtomicIncrement( volatile intptr_t& counter ) { + return __TBB_FetchAndAddW( &counter, 1 )+1; +} + +inline uintptr_t AtomicAdd( volatile intptr_t& counter, intptr_t value ) { + return __TBB_FetchAndAddW( &counter, value ); +} + +inline intptr_t AtomicCompareExchange( volatile intptr_t& location, intptr_t new_value, intptr_t comparand) { + return __TBB_CompareAndSwapW( &location, new_value, comparand ); +} + +inline uintptr_t AtomicFetchStore(volatile void* location, uintptr_t value) { + return __TBB_FetchAndStoreW(location, value); +} + +inline void AtomicOr(volatile void *operand, uintptr_t addend) { + __TBB_AtomicOR(operand, addend); +} + +inline void AtomicAnd(volatile void *operand, uintptr_t addend) { + __TBB_AtomicAND(operand, addend); +} + +inline intptr_t FencedLoad( const volatile intptr_t &location ) { + return __TBB_load_with_acquire(location); +} + +inline void FencedStore( volatile intptr_t &location, intptr_t value ) { + __TBB_store_with_release(location, value); +} + +inline void SpinWaitWhileEq(const volatile intptr_t &location, const intptr_t value) { + tbb::internal::spin_wait_while_eq(location, value); +} + +class AtomicBackoff { + tbb::internal::atomic_backoff backoff; +public: + AtomicBackoff() {} + void pause() { backoff.pause(); } +}; + +inline void SpinWaitUntilEq(const volatile intptr_t &location, const intptr_t value) { + tbb::internal::spin_wait_until_eq(location, value); +} + +#endif /* __TBB_malloc_Synchronize_H_ */ diff --git a/src/tbb/src/tbbmalloc/TypeDefinitions.h b/src/tbb/src/tbbmalloc/TypeDefinitions.h index 9b514ca4..3178442e 100644 --- a/src/tbb/src/tbbmalloc/TypeDefinitions.h +++ b/src/tbb/src/tbbmalloc/TypeDefinitions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef _itt_shared_malloc_TypeDefinitions_H_ diff --git a/src/tbb/src/tbbmalloc/backend.cpp b/src/tbb/src/tbbmalloc/backend.cpp index 88212dd6..a93ac74b 100644 --- a/src/tbb/src/tbbmalloc/backend.cpp +++ b/src/tbb/src/tbbmalloc/backend.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include /* for memset */ @@ -43,60 +39,14 @@ namespace internal { /* assume MapMemory and UnmapMemory are customized */ #endif -void* getRawMemory (size_t size, bool hugePages) { - return MapMemory(size, hugePages); +void* getRawMemory (size_t size, PageType pageType) { + return MapMemory(size, pageType); } int freeRawMemory (void *object, size_t size) { return UnmapMemory(object, size); } -void HugePagesStatus::registerAllocation(bool gotPage) -{ - if (gotPage) { - if (!wasObserved) - FencedStore(wasObserved, 1); - } else - FencedStore(enabled, 0); - // reports huge page status only once - if (needActualStatusPrint - && AtomicCompareExchange(needActualStatusPrint, 0, 1)) - doPrintStatus(gotPage, "available"); -} - -void HugePagesStatus::registerReleasing(void* addr, size_t size) -{ - // We: 1) got huge page at least once, - // 2) something that looks like a huge page is been released, - // and 3) user requested huge pages, - // so a huge page might be available at next allocation. - // TODO: keep page status in regions and use exact check here - if (FencedLoad(wasObserved) && size>=pageSize && isAligned(addr, pageSize)) - FencedStore(enabled, requestedMode.get()); -} - -void HugePagesStatus::printStatus() { - doPrintStatus(requestedMode.get(), "requested"); - if (requestedMode.get()) { // report actual status iff requested - if (pageSize) - FencedStore(needActualStatusPrint, 1); - else - doPrintStatus(/*state=*/false, "available"); - } -} - -void HugePagesStatus::doPrintStatus(bool state, const char *stateName) -{ - // Under macOS* fprintf/snprintf acquires an internal lock, so when - // 1st allocation is done under the lock, we got a deadlock. - // Do not use fprintf etc during initialization. - fputs("TBBmalloc: huge pages\t", stderr); - if (!state) - fputs("not ", stderr); - fputs(stateName, stderr); - fputs("\n", stderr); -} - #if CHECK_ALLOCATION_RANGE void Backend::UsedAddressRange::registerAlloc(uintptr_t left, uintptr_t right) @@ -127,34 +77,43 @@ void Backend::UsedAddressRange::registerFree(uintptr_t left, uintptr_t right) } #endif // CHECK_ALLOCATION_RANGE +// Initialized in frontend inside defaultMemPool +extern HugePagesStatus hugePages; + void *Backend::allocRawMem(size_t &size) { void *res = NULL; - size_t allocSize; + size_t allocSize = 0; if (extMemPool->userPool()) { - if (extMemPool->fixedPool && bootsrapMemDone==FencedLoad(bootsrapMemStatus)) + if (extMemPool->fixedPool && bootsrapMemDone == FencedLoad(bootsrapMemStatus)) return NULL; - MALLOC_ASSERT(bootsrapMemStatus!=bootsrapMemNotDone, + MALLOC_ASSERT(bootsrapMemStatus != bootsrapMemNotDone, "Backend::allocRawMem() called prematurely?"); // TODO: support for raw mem not aligned at sizeof(uintptr_t) // memory from fixed pool is asked once and only once allocSize = alignUpGeneric(size, extMemPool->granularity); res = (*extMemPool->rawAlloc)(extMemPool->poolId, allocSize); } else { - // check if alignment to huge page size is recommended - size_t hugePageSize = hugePages.recommendedGranularity(); - allocSize = alignUpGeneric(size, hugePageSize? hugePageSize : extMemPool->granularity); - // try to get them at 1st allocation and still use, if successful - // if 1st try is unsuccessful, no more trying - if (FencedLoad(hugePages.enabled)) { - MALLOC_ASSERT(hugePageSize, "Inconsistent state of HugePagesStatus"); - res = getRawMemory(allocSize, /*hugePages=*/true); - hugePages.registerAllocation(res); + // Align allocation on page size + size_t pageSize = hugePages.isEnabled ? hugePages.getGranularity() : extMemPool->granularity; + MALLOC_ASSERT(pageSize, "Page size cannot be zero."); + allocSize = alignUpGeneric(size, pageSize); + + // If user requested huge pages and they are available, try to use preallocated ones firstly. + // If there are none, lets check transparent huge pages support and use them instead. + if (hugePages.isEnabled) { + if (hugePages.isHPAvailable) { + res = getRawMemory(allocSize, PREALLOCATED_HUGE_PAGE); + } + if (!res && hugePages.isTHPAvailable) { + res = getRawMemory(allocSize, TRANSPARENT_HUGE_PAGE); + } } - if (!res) - res = getRawMemory(allocSize, /*hugePages=*/false); + if (!res) { + res = getRawMemory(allocSize, REGULAR); + } } if (res) { @@ -185,7 +144,6 @@ bool Backend::freeRawMem(void *object, size_t size) fail = (*extMemPool->rawFree)(extMemPool->poolId, object, size); } else { usedAddrRange.registerFree((uintptr_t)object, (uintptr_t)object + size); - hugePages.registerReleasing(object, size); fail = freeRawMemory(object, size); } // TODO: use result in all freeRawMem() callers @@ -263,7 +221,7 @@ class FreeBlock : BlockMutexes { // valid only when block is in processing, i.e. one is not free and not size_t sizeTmp; // used outside of backend int myBin; // bin that is owner of the block - bool aligned; + bool slabAligned; bool blockInBin; // this block in myBin already FreeBlock *rightNeig(size_t sz) const { @@ -419,9 +377,8 @@ inline void CoalRequestQ::blockWasProcessed() // split the block without removing it. // If the free space should go to other bin(s), remove the block. // alignedBin is true, if all blocks in the bin have slab-aligned right side. -FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, - size_t size, bool needAlignedRes, bool alignedBin, bool wait, - int *binLocked) +FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, size_t size, + bool needAlignedRes, bool alignedBin, bool wait, int *binLocked) { Bin *b = &freeBins[binIdx]; try_next: @@ -443,50 +400,36 @@ FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, goto try_next; } + // GENERAL CASE if (alignedBin || !needAlignedRes) { size_t splitSz = szBlock - size; - // If we got a block as split result, - // it must have a room for control structures. - if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || - !splitSz)) + // If we got a block as split result, it must have a room for control structures. + if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || !splitSz)) fBlock = curr; } else { + // SPECIAL CASE, to get aligned block from unaligned bin we have to cut the middle of a block + // and return remaining left and right part. Possible only in fixed pool scenario, assert for this + // is set inside splitBlock() function. + void *newB = alignUp(curr, slabSize); uintptr_t rightNew = (uintptr_t)newB + size; uintptr_t rightCurr = (uintptr_t)curr + szBlock; - // appropriate size, and left and right split results - // are either big enough or non-existent + // Check if the block size is sufficient, + // and also left and right split results are either big enough or non-existent if (rightNew <= rightCurr - && (newB==curr || - (uintptr_t)newB-(uintptr_t)curr >= FreeBlock::minBlockSize) - && (rightNew==rightCurr || - rightCurr - rightNew >= FreeBlock::minBlockSize)) + && (newB == curr || ((uintptr_t)newB - (uintptr_t)curr) >= FreeBlock::minBlockSize) + && (rightNew == rightCurr || (rightCurr - rightNew) >= FreeBlock::minBlockSize)) fBlock = curr; } + if (fBlock) { - // consume must be called before result of removing from a bin - // is visible externally. + // consume must be called before result of removing from a bin is visible externally. sync->blockConsumed(); - if (alignedBin && needAlignedRes && - Backend::sizeToBin(szBlock-size) == Backend::sizeToBin(szBlock)) { - // free remainder of fBlock stay in same bin, - // so no need to remove it from the bin - // TODO: add more "still here" cases - FreeBlock *newFBlock = fBlock; - // return block from right side of fBlock - fBlock = (FreeBlock*)((uintptr_t)newFBlock + szBlock - size); - MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block"); - fBlock->initHeader(); - fBlock->setLeftFree(szBlock - size); - newFBlock->setMeFree(szBlock - size); - - fBlock->sizeTmp = size; - } else { - b->removeBlock(fBlock); - if (freeBins[binIdx].empty()) - bitMask.set(binIdx, false); - fBlock->sizeTmp = szBlock; - } + // TODO: think about cases when block stays in the same bin + b->removeBlock(fBlock); + if (freeBins[binIdx].empty()) + bitMask.set(binIdx, false); + fBlock->sizeTmp = szBlock; break; } else { // block size is not valid, search for next block in the bin curr->setMeFree(szBlock); @@ -542,9 +485,7 @@ void Backend::Bin::removeBlock(FreeBlock *fBlock) void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail) { Bin *b = &freeBins[binIdx]; - fBlock->myBin = binIdx; - fBlock->aligned = toAlignedBin(fBlock, blockSz); fBlock->next = fBlock->prev = NULL; { MallocMutex::scoped_lock scopedLock(b->tLock); @@ -571,9 +512,7 @@ bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addTo { bool locked; Bin *b = &freeBins[binIdx]; - fBlock->myBin = binIdx; - fBlock->aligned = toAlignedBin(fBlock, fBlock->sizeTmp); if (addToTail) { fBlock->next = NULL; { @@ -625,77 +564,65 @@ bool ExtMemoryPool::regionsAreReleaseable() const return !keepAllMemory && !delayRegsReleasing; } -FreeBlock *Backend::splitUnalignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedBlock) +FreeBlock *Backend::splitBlock(FreeBlock *fBlock, int num, size_t size, bool blockIsAligned, bool needAlignedBlock) { - const size_t totalSize = num*size; - if (needAlignedBlock) { - size_t fBlockSz = fBlock->sizeTmp; - uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlockSz; - FreeBlock *newB = alignUp(fBlock, slabSize); - FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newB + totalSize); - - // Space to use is in the middle, - // ... return free right part + const size_t totalSize = num * size; + + // SPECIAL CASE, for unaligned block we have to cut the middle of a block + // and return remaining left and right part. Possible only in a fixed pool scenario. + if (needAlignedBlock && !blockIsAligned) { + MALLOC_ASSERT(extMemPool->fixedPool, + "Aligned block request from unaligned bin possible only in fixed pool scenario."); + + // Space to use is in the middle + FreeBlock *newBlock = alignUp(fBlock, slabSize); + FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newBlock + totalSize); + uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlock->sizeTmp; + + // Return free right part if ((uintptr_t)rightPart != fBlockEnd) { rightPart->initHeader(); // to prevent coalescing rightPart with fBlock - coalescAndPut(rightPart, fBlockEnd - (uintptr_t)rightPart); - } - // ... and free left part - if (newB != fBlock) { - newB->initHeader(); // to prevent coalescing fBlock with newB - coalescAndPut(fBlock, (uintptr_t)newB - (uintptr_t)fBlock); + size_t rightSize = fBlockEnd - (uintptr_t)rightPart; + coalescAndPut(rightPart, rightSize, toAlignedBin(rightPart, rightSize)); } - - fBlock = newB; - MALLOC_ASSERT(isAligned(fBlock, slabSize), ASSERT_TEXT); - } else { - if (size_t splitSz = fBlock->sizeTmp - totalSize) { - // split block and return free right part - FreeBlock *splitB = (FreeBlock*)((uintptr_t)fBlock + totalSize); - splitB->initHeader(); - coalescAndPut(splitB, splitSz); + // And free left part + if (newBlock != fBlock) { + newBlock->initHeader(); // to prevent coalescing fBlock with newB + size_t leftSize = (uintptr_t)newBlock - (uintptr_t)fBlock; + coalescAndPut(fBlock, leftSize, toAlignedBin(fBlock, leftSize)); } - } - FreeBlock::markBlocks(fBlock, num, size); - return fBlock; -} - -FreeBlock *Backend::splitAlignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedBlock) -{ - if (fBlock->sizeTmp != num*size) { // i.e., need to split the block - FreeBlock *newAlgnd; - size_t newSz; - + fBlock = newBlock; + } else if (size_t splitSize = fBlock->sizeTmp - totalSize) { // need to split the block + // GENERAL CASE, cut the left or right part of the block + FreeBlock *splitBlock = NULL; if (needAlignedBlock) { - newAlgnd = fBlock; - fBlock = (FreeBlock*)((uintptr_t)newAlgnd + newAlgnd->sizeTmp - - num*size); - MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block"); + // For slab aligned blocks cut the right side of the block + // and return it to a requester, original block returns to backend + splitBlock = fBlock; + fBlock = (FreeBlock*)((uintptr_t)splitBlock + splitSize); fBlock->initHeader(); - newSz = newAlgnd->sizeTmp - num*size; } else { - newAlgnd = (FreeBlock*)((uintptr_t)fBlock + num*size); - newSz = fBlock->sizeTmp - num*size; - newAlgnd->initHeader(); + // For large object blocks cut original block and put free righ part to backend + splitBlock = (FreeBlock*)((uintptr_t)fBlock + totalSize); + splitBlock->initHeader(); } - coalescAndPut(newAlgnd, newSz); + // Mark free block as it`s parent only when the requested type (needAlignedBlock) + // and returned from Bins/OS block (isAligned) are equal (XOR operation used) + bool markAligned = (blockIsAligned ^ needAlignedBlock) ? toAlignedBin(splitBlock, splitSize) : blockIsAligned; + coalescAndPut(splitBlock, splitSize, markAligned); } - MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), - "Expect to get aligned block, if one was requested."); + MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), "Expect to get aligned block, if one was requested."); FreeBlock::markBlocks(fBlock, num, size); return fBlock; } -inline size_t Backend::getMaxBinnedSize() const +size_t Backend::getMaxBinnedSize() const { - return hugePages.wasObserved && !inUserPool()? + return hugePages.isEnabled && !inUserPool() ? maxBinned_HugePage : maxBinned_SmallPage; } -inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, - size_t requestSize) const +inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, size_t requestSize) const { return requestSize > oldMaxReq && requestSize < backend->getMaxBinnedSize(); } @@ -719,7 +646,7 @@ FreeBlock *Backend::releaseMemInCaches(intptr_t startModifiedCnt, FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins, - bool *splittableRet) + bool *splittableRet, bool needSlabRegion) { FreeBlock *block; // The block sizes can be divided into 3 groups: @@ -768,20 +695,21 @@ FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt, // This must be done carefully, because blocks in bins can be released // in releaseCachesToLimit(). const unsigned NUM_OF_REG = 3; - block = addNewRegion(regSz_sizeBased, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/false); + MemRegionType regType = needSlabRegion ? MEMREG_SLAB_BLOCKS : MEMREG_LARGE_BLOCKS; + block = addNewRegion(regSz_sizeBased, regType, /*addToBin=*/false); if (block) for (unsigned idx=0; idxallLocalCaches.cleanup(extMemPool, /*cleanOnlyUnused=*/true) - : extMemPool->loc.decreasingCleanup()) + while (cleanLocal ? + extMemPool->allLocalCaches.cleanup(/*cleanOnlyUnused=*/true) : + extMemPool->loc.decreasingCleanup()) if ((locTotalMemSize = FencedLoad((intptr_t&)totalMemSize)) <= (locMemSoftLimit = FencedLoad((intptr_t&)memSoftLimit))) return; @@ -815,13 +743,17 @@ void Backend::releaseCachesToLimit() extMemPool->hardCachesCleanup(); } -FreeBlock *Backend::IndexedBins:: - findBlock(int nativeBin, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, int *numOfLockedBins) +int Backend::IndexedBins::getMinNonemptyBin(unsigned startBin) const +{ + int p = bitMask.getMinTrue(startBin); + return p == -1 ? Backend::freeBinsNum : p; +} + +FreeBlock *Backend::IndexedBins::findBlock(int nativeBin, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin, int *numOfLockedBins) { for (int i=getMinNonemptyBin(nativeBin); ifixedPool) + block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/false, &numOfLockedBins); } else { - block = freeLargeBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/false, /*alignedBin=*/false, - &numOfLockedBins); - if (!block) - block = freeAlignedBins.findBlock(nativeBin, &bkndSync, num*size, - /*needAlignedBlock=*/false, /*alignedBin=*/true, - &numOfLockedBins); + block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/false, &numOfLockedBins); + if (!block && extMemPool->fixedPool) + block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock, + /*alignedBin=*/true, &numOfLockedBins); } } while (!block && numOfLockedBins>lockedBinsThreshold); if (block) break; - if (!(scanCoalescQ(/*forceCoalescQDrop=*/true) - | extMemPool->softCachesCleanup())) { + if (!(scanCoalescQ(/*forceCoalescQDrop=*/true) | extMemPool->softCachesCleanup())) { // bins are not updated, // only remaining possibility is to ask for more memory - block = - askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold, - numOfLockedBins, &splittable); + block = askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold, + numOfLockedBins, &splittable, needAlignedBlock); if (!block) return NULL; if (block != (FreeBlock*)VALID_BLOCK_IN_BIN) { @@ -911,10 +835,10 @@ FreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock) } } MALLOC_ASSERT(block, ASSERT_TEXT); - if (splittable) - block = toAlignedBin(block, block->sizeTmp)? - splitAlignedBlock(block, num, size, needAlignedBlock) : - splitUnalignedBlock(block, num, size, needAlignedBlock); + if (splittable) { + // At this point we have to be sure that slabAligned attribute describes the right block state + block = splitBlock(block, num, size, block->slabAligned, needAlignedBlock); + } // matched blockConsumed() from startUseBlock() bkndSync.blockReleased(); @@ -933,12 +857,22 @@ LargeMemoryBlock *Backend::getLargeBlock(size_t size) return lmb; } +BlockI *Backend::getSlabBlock(int num) { + BlockI *b = (BlockI*)genericGetBlock(num, slabSize, /*slabAligned=*/true); + MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT); + return b; +} + +void Backend::putSlabBlock(BlockI *block) { + genericPutBlock((FreeBlock *)block, slabSize, /*slabAligned=*/true); +} + void *Backend::getBackRefSpace(size_t size, bool *rawMemUsed) { // This block is released only at shutdown, so it can prevent // a entire region releasing when it's received from the backend, // so prefer getRawMemory using. - if (void *ret = getRawMemory(size, /*hugePages=*/false)) { + if (void *ret = getRawMemory(size, REGULAR)) { *rawMemUsed = true; return ret; } @@ -957,17 +891,17 @@ void Backend::putBackRefSpace(void *b, size_t size, bool rawMemUsed) void Backend::removeBlockFromBin(FreeBlock *fBlock) { if (fBlock->myBin != Backend::NO_BIN) { - if (fBlock->aligned) - freeAlignedBins.lockRemoveBlock(fBlock->myBin, fBlock); + if (fBlock->slabAligned) + freeSlabAlignedBins.lockRemoveBlock(fBlock->myBin, fBlock); else - freeLargeBins.lockRemoveBlock(fBlock->myBin, fBlock); + freeLargeBlockBins.lockRemoveBlock(fBlock->myBin, fBlock); } } -void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz) +void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned) { bkndSync.blockConsumed(); - coalescAndPut(fBlock, blockSz); + coalescAndPut(fBlock, blockSz, slabAligned); bkndSync.blockReleased(); } @@ -996,7 +930,7 @@ void Backend::putLargeBlock(LargeMemoryBlock *lmb) { if (extMemPool->userPool()) extMemPool->lmbList.remove(lmb); - genericPutBlock((FreeBlock *)lmb, lmb->unalignedSize); + genericPutBlock((FreeBlock *)lmb, lmb->unalignedSize, false); } void Backend::returnLargeObject(LargeMemoryBlock *lmb) @@ -1030,9 +964,10 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment if (oldRegion->type != MEMREG_ONE_BLOCK) return NULL; // we are not single in the region const size_t userOffset = (uintptr_t)ptr - (uintptr_t)oldRegion; + const size_t alignedSize = LargeObjectCache::alignToBin(newSize + userOffset); const size_t requestSize = - alignUp(userOffset + newSize + sizeof(LastFreeBlock), extMemPool->granularity); - if (requestSize < newSize) // is wrapped around? + alignUp(sizeof(MemRegion) + alignedSize + sizeof(LastFreeBlock), extMemPool->granularity); + if (requestSize < alignedSize) // is wrapped around? return NULL; regionList.remove(oldRegion); @@ -1044,12 +979,10 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment MemRegion *region = (MemRegion*)ret; MALLOC_ASSERT(region->type == MEMREG_ONE_BLOCK, ASSERT_TEXT); region->allocSz = requestSize; + region->blockSz = alignedSize; FreeBlock *fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment); - // put LastFreeBlock at the very end of region - const uintptr_t fBlockEnd = (uintptr_t)region + requestSize - sizeof(LastFreeBlock); - region->blockSz = fBlockEnd - (uintptr_t)fBlock; regionList.add(region); startUseBlock(region, fBlock, /*addToBin=*/false); @@ -1058,7 +991,7 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment // TODO: get rid of useless pair blockConsumed()/blockReleased() bkndSync.blockReleased(); - // object must start at same offest from region's start + // object must start at same offset from region's start void *object = (void*)((uintptr_t)region + userOffset); MALLOC_ASSERT(isAligned(object, alignment), ASSERT_TEXT); LargeObjectHdr *header = (LargeObjectHdr*)object - 1; @@ -1173,8 +1106,7 @@ FreeBlock *Backend::doCoalesc(FreeBlock *fBlock, MemRegion **mRegion) return resBlock; } -bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, - bool reportBlocksProcessed) +bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, bool reportBlocksProcessed) { bool regionReleased = false; @@ -1204,12 +1136,12 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, } size_t currSz = toRet->sizeTmp; int bin = sizeToBin(currSz); - bool toAligned = toAlignedBin(toRet, currSz); + bool toAligned = extMemPool->fixedPool ? toAlignedBin(toRet, currSz) : toRet->slabAligned; bool needAddToBin = true; if (toRet->blockInBin) { // Does it stay in same bin? - if (toRet->myBin == bin && toRet->aligned == toAligned) + if (toRet->myBin == bin && toRet->slabAligned == toAligned) needAddToBin = false; else { toRet->blockInBin = false; @@ -1221,12 +1153,13 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, if (needAddToBin) { toRet->prev = toRet->next = toRet->nextToFree = NULL; toRet->myBin = NO_BIN; + toRet->slabAligned = toAligned; // If the block is too small to fit in any bin, keep it bin-less. // It's not a leak because the block later can be coalesced. if (currSz >= minBinnedSize) { toRet->sizeTmp = currSz; - IndexedBins *target = toAligned? &freeAlignedBins : &freeLargeBins; + IndexedBins *target = toRet->slabAligned ? &freeSlabAlignedBins : &freeLargeBlockBins; if (forceCoalescQDrop) { target->addBlock(bin, toRet, toRet->sizeTmp, addToTail); } else if (!target->tryAddBlock(bin, toRet, addToTail)) { @@ -1249,10 +1182,11 @@ bool Backend::coalescAndPutList(FreeBlock *list, bool forceCoalescQDrop, // Coalesce fBlock and add it back to a bin; // processing delayed coalescing requests. -void Backend::coalescAndPut(FreeBlock *fBlock, size_t blockSz) +void Backend::coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned) { fBlock->sizeTmp = blockSz; fBlock->nextToFree = NULL; + fBlock->slabAligned = slabAligned; coalescAndPutList(fBlock, /*forceCoalescQDrop=*/false, /*reportBlocksProcessed=*/false); } @@ -1266,7 +1200,7 @@ bool Backend::scanCoalescQ(bool forceCoalescQDrop) // matches blockConsumed() from CoalRequestQ::putBlock() coalescAndPutList(currCoalescList, forceCoalescQDrop, /*reportBlocksProcessed=*/true); - // returns status of coalescQ.getAll(), as an indication of possibe changes in backend + // returns status of coalescQ.getAll(), as an indication of possible changes in backend // TODO: coalescAndPutList() may report is some new free blocks became available or not return currCoalescList; } @@ -1282,13 +1216,11 @@ FreeBlock *Backend::findBlockInRegion(MemRegion *region, size_t exactBlockSize) "Atomic applied on LastFreeBlock, and we put it at the end of region, that" " is uintptr_t-aligned, so no unaligned atomic operations are possible."); // right bound is slab-aligned, keep LastFreeBlock after it - if (region->type==MEMREG_FLEXIBLE_SIZE) { - fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), - sizeof(uintptr_t)); + if (region->type == MEMREG_SLAB_BLOCKS) { + fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), sizeof(uintptr_t)); fBlockEnd = alignDown(lastFreeBlock, slabSize); } else { - fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), - largeObjectAlignment); + fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment); fBlockEnd = (uintptr_t)fBlock + exactBlockSize; MALLOC_ASSERT(fBlockEnd <= lastFreeBlock, ASSERT_TEXT); } @@ -1305,7 +1237,7 @@ FreeBlock *Backend::findBlockInRegion(MemRegion *region, size_t exactBlockSize) return fBlock; } -// startUseBlock adds free block to a bin, the block can be used and +// startUseBlock may add the free block to a bin, the block can be used and // even released after this, so the region must be added to regionList already void Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin) { @@ -1326,17 +1258,20 @@ void Backend::startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin) unsigned targetBin = sizeToBin(blockSz); // during adding advance regions, register bin for a largest block in region advRegBins.registerBin(targetBin); - if (region->type!=MEMREG_ONE_BLOCK && toAlignedBin(fBlock, blockSz)) { - freeAlignedBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); + if (region->type == MEMREG_SLAB_BLOCKS) { + fBlock->slabAligned = true; + freeSlabAlignedBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); } else { - freeLargeBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); + fBlock->slabAligned = false; + freeLargeBlockBins.addBlock(targetBin, fBlock, blockSz, /*addToTail=*/false); } } else { // to match with blockReleased() in genericGetBlock bkndSync.blockConsumed(); + // Understand our alignment for correct splitBlock operation + fBlock->slabAligned = region->type == MEMREG_SLAB_BLOCKS ? true : false; fBlock->sizeTmp = fBlock->tryLockBlock(); - MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, - "Locking must be successful"); + MALLOC_ASSERT(fBlock->sizeTmp >= FreeBlock::minBlockSize, "Locking must be successful"); } } @@ -1380,9 +1315,9 @@ FreeBlock *Backend::addNewRegion(size_t size, MemRegionType memRegType, bool add "Header must be not overwritten in used blocks"); MALLOC_ASSERT(FreeBlock::minBlockSize > GuardedSize::MAX_SPEC_VAL, "Block length must not conflict with special values of GuardedSize"); - // If the region is not "flexible size" we should reserve some space for + // If the region is not "for slabs" we should reserve some space for // a region header, the worst case alignment and the last block mark. - const size_t requestSize = memRegType == MEMREG_FLEXIBLE_SIZE ? size : + const size_t requestSize = memRegType == MEMREG_SLAB_BLOCKS ? size : size + sizeof(MemRegion) + largeObjectAlignment + FreeBlock::minBlockSize + sizeof(LastFreeBlock); @@ -1426,8 +1361,8 @@ void Backend::reset() // no active threads are allowed in backend while reset() called verify(); - freeLargeBins.reset(); - freeAlignedBins.reset(); + freeLargeBlockBins.reset(); + freeSlabAlignedBins.reset(); advRegBins.reset(); for (MemRegion *curr = regionList.head; curr; curr = curr->next) { @@ -1443,8 +1378,8 @@ bool Backend::destroy() // no active threads are allowed in backend while destroy() called verify(); if (!inUserPool()) { - freeLargeBins.reset(); - freeAlignedBins.reset(); + freeLargeBlockBins.reset(); + freeSlabAlignedBins.reset(); } while (regionList.head) { MemRegion *helper = regionList.head->next; @@ -1463,10 +1398,10 @@ bool Backend::clean() // because such regions are added in advance (see askMemFromOS() and reset()), // and never used. Release them all. for (int i = advRegBins.getMinUsedBin(0); i != -1; i = advRegBins.getMinUsedBin(i+1)) { - if (i == freeAlignedBins.getMinNonemptyBin(i)) - res |= freeAlignedBins.tryReleaseRegions(i, this); - if (i == freeLargeBins.getMinNonemptyBin(i)) - res |= freeLargeBins.tryReleaseRegions(i, this); + if (i == freeSlabAlignedBins.getMinNonemptyBin(i)) + res |= freeSlabAlignedBins.tryReleaseRegions(i, this); + if (i == freeLargeBlockBins.getMinNonemptyBin(i)) + res |= freeLargeBlockBins.tryReleaseRegions(i, this); } return res; @@ -1494,8 +1429,8 @@ void Backend::verify() #if MALLOC_DEBUG scanCoalescQ(/*forceCoalescQDrop=*/false); - freeLargeBins.verify(); - freeAlignedBins.verify(); + freeLargeBlockBins.verify(); + freeSlabAlignedBins.verify(); #endif // MALLOC_DEBUG } @@ -1544,9 +1479,9 @@ void Backend::reportStat(FILE *f) int regNum = regionList.reportStat(f); fprintf(f, "\n%d regions, %lu KB in all regions\n free bins:\nlarge bins: ", regNum, totalMemSize/1024); - freeLargeBins.reportStat(f); + freeLargeBlockBins.reportStat(f); fprintf(f, "\naligned bins: "); - freeAlignedBins.reportStat(f); + freeSlabAlignedBins.reportStat(f); fprintf(f, "\n"); } #endif // __TBB_MALLOC_BACKEND_STAT diff --git a/src/tbb/src/tbbmalloc/backend.h b/src/tbb/src/tbbmalloc/backend.h new file mode 100644 index 00000000..981086d2 --- /dev/null +++ b/src/tbb/src/tbbmalloc/backend.h @@ -0,0 +1,385 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + +#ifndef __TBB_backend_H +#define __TBB_backend_H + +// Included from namespace rml::internal + +// global state of blocks currently in processing +class BackendSync { + // Class instances should reside in zero-initialized memory! + // The number of blocks currently removed from a bin and not returned back + intptr_t inFlyBlocks; // to another + intptr_t binsModifications; // incremented on every bin modification + Backend *backend; +public: + void init(Backend *b) { backend = b; } + void blockConsumed() { AtomicIncrement(inFlyBlocks); } + void binsModified() { AtomicIncrement(binsModifications); } + void blockReleased() { +#if __TBB_MALLOC_BACKEND_STAT + MALLOC_ITT_SYNC_RELEASING(&inFlyBlocks); +#endif + AtomicIncrement(binsModifications); + intptr_t prev = AtomicAdd(inFlyBlocks, -1); + MALLOC_ASSERT(prev > 0, ASSERT_TEXT); + suppress_unused_warning(prev); + } + intptr_t getNumOfMods() const { return FencedLoad(binsModifications); } + // return true if need re-do the blocks search + inline bool waitTillBlockReleased(intptr_t startModifiedCnt); +}; + +class CoalRequestQ { // queue of free blocks that coalescing was delayed +private: + FreeBlock *blocksToFree; + BackendSync *bkndSync; + // counted blocks in blocksToFree and that are leaved blocksToFree + // and still in active coalescing + intptr_t inFlyBlocks; +public: + void init(BackendSync *bSync) { bkndSync = bSync; } + FreeBlock *getAll(); // return current list of blocks and make queue empty + void putBlock(FreeBlock *fBlock); + inline void blockWasProcessed(); + intptr_t blocksInFly() const { return FencedLoad(inFlyBlocks); } +}; + +class MemExtendingSema { + intptr_t active; +public: + bool wait() { + bool rescanBins = false; + // up to 3 threads can add more memory from OS simultaneously, + // rest of threads have to wait + for (;;) { + intptr_t prevCnt = FencedLoad(active); + if (prevCnt < 3) { + intptr_t n = AtomicCompareExchange(active, prevCnt+1, prevCnt); + if (n == prevCnt) + break; + } else { + SpinWaitWhileEq(active, prevCnt); + rescanBins = true; + break; + } + } + return rescanBins; + } + void signal() { AtomicAdd(active, -1); } +}; + +enum MemRegionType { + // The region holds only slabs + MEMREG_SLAB_BLOCKS = 0, + // The region can hold several large object blocks + MEMREG_LARGE_BLOCKS, + // The region holds only one block with a requested size + MEMREG_ONE_BLOCK +}; + +class MemRegionList { + MallocMutex regionListLock; +public: + MemRegion *head; + void add(MemRegion *r); + void remove(MemRegion *r); + int reportStat(FILE *f); +}; + +class Backend { +private: +/* Blocks in range [minBinnedSize; getMaxBinnedSize()] are kept in bins, + one region can contains several blocks. Larger blocks are allocated directly + and one region always contains one block. +*/ + enum { + minBinnedSize = 8*1024UL, + /* If huge pages are available, maxBinned_HugePage used. + If not, maxBinned_SmallPage is the threshold. + TODO: use pool's granularity for upper bound setting.*/ + maxBinned_SmallPage = 1024*1024UL, + // TODO: support other page sizes + maxBinned_HugePage = 4*1024*1024UL + }; + enum { + VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result + }; +public: + // Backend bins step is the same as CacheStep for large object cache + static const size_t freeBinsStep = LargeObjectCache::LargeBSProps::CacheStep; + static const unsigned freeBinsNum = (maxBinned_HugePage-minBinnedSize)/freeBinsStep + 1; + + // if previous access missed per-thread slabs pool, + // allocate numOfSlabAllocOnMiss blocks in advance + static const int numOfSlabAllocOnMiss = 2; + + enum { + NO_BIN = -1, + // special bin for blocks >= maxBinned_HugePage, blocks go to this bin + // when pool is created with keepAllMemory policy + // TODO: currently this bin is scanned using "1st fit", as it accumulates + // blocks of different sizes, "best fit" is preferred in terms of fragmentation + HUGE_BIN = freeBinsNum-1 + }; + + // Bin keeps 2-linked list of free blocks. It must be 2-linked + // because during coalescing a block it's removed from a middle of the list. + struct Bin { + FreeBlock *head, + *tail; + MallocMutex tLock; + + void removeBlock(FreeBlock *fBlock); + void reset() { head = tail = 0; } + bool empty() const { return !head; } + + size_t countFreeBlocks(); + size_t reportFreeBlocks(FILE *f); + void reportStat(FILE *f); + }; + + typedef BitMaskMin BitMaskBins; + + // array of bins supplemented with bitmask for fast finding of non-empty bins + class IndexedBins { + BitMaskBins bitMask; + Bin freeBins[Backend::freeBinsNum]; + FreeBlock *getFromBin(int binIdx, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin, bool wait, int *resLocked); + public: + FreeBlock *findBlock(int nativeBin, BackendSync *sync, size_t size, + bool needAlignedBlock, bool alignedBin,int *numOfLockedBins); + bool tryReleaseRegions(int binIdx, Backend *backend); + void lockRemoveBlock(int binIdx, FreeBlock *fBlock); + void addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail); + bool tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail); + int getMinNonemptyBin(unsigned startBin) const; + void verify(); + void reset(); + void reportStat(FILE *f); + }; + +private: + class AdvRegionsBins { + BitMaskBins bins; + public: + void registerBin(int regBin) { bins.set(regBin, 1); } + int getMinUsedBin(int start) const { return bins.getMinTrue(start); } + void reset() { bins.reset(); } + }; + // auxiliary class to atomic maximum request finding + class MaxRequestComparator { + const Backend *backend; + public: + MaxRequestComparator(const Backend *be) : backend(be) {} + inline bool operator()(size_t oldMaxReq, size_t requestSize) const; + }; + +#if CHECK_ALLOCATION_RANGE + // Keep min and max of all addresses requested from OS, + // use it for checking memory possibly allocated by replaced allocators + // and for debugging purposes. Valid only for default memory pool. + class UsedAddressRange { + static const uintptr_t ADDRESS_UPPER_BOUND = UINTPTR_MAX; + + uintptr_t leftBound, + rightBound; + MallocMutex mutex; + public: + // rightBound is zero-initialized + void init() { leftBound = ADDRESS_UPPER_BOUND; } + void registerAlloc(uintptr_t left, uintptr_t right); + void registerFree(uintptr_t left, uintptr_t right); + // as only left and right bounds are kept, we can return true + // for pointer not allocated by us, if more than single region + // was requested from OS + bool inRange(void *ptr) const { + const uintptr_t p = (uintptr_t)ptr; + return leftBound<=p && p<=rightBound; + } + }; +#else + class UsedAddressRange { + public: + void init() { } + void registerAlloc(uintptr_t, uintptr_t) {} + void registerFree(uintptr_t, uintptr_t) {} + bool inRange(void *) const { return true; } + }; +#endif + + ExtMemoryPool *extMemPool; + // used for release every region on pool destroying + MemRegionList regionList; + + CoalRequestQ coalescQ; // queue of coalescing requests + BackendSync bkndSync; + // semaphore protecting adding more more memory from OS + MemExtendingSema memExtendingSema; + size_t totalMemSize, + memSoftLimit; + UsedAddressRange usedAddrRange; + // to keep 1st allocation large than requested, keep bootstrapping status + enum { + bootsrapMemNotDone = 0, + bootsrapMemInitializing, + bootsrapMemDone + }; + intptr_t bootsrapMemStatus; + MallocMutex bootsrapMemStatusMutex; + + // Using of maximal observed requested size allows decrease + // memory consumption for small requests and decrease fragmentation + // for workloads when small and large allocation requests are mixed. + // TODO: decrease, not only increase it + size_t maxRequestedSize; + + // register bins related to advance regions + AdvRegionsBins advRegBins; + // Storage for split FreeBlocks + IndexedBins freeLargeBlockBins, + freeSlabAlignedBins; + + // Our friends + friend class BackendSync; + + /******************************** Backend methods ******************************/ + + /*--------------------------- Coalescing functions ----------------------------*/ + void coalescAndPut(FreeBlock *fBlock, size_t blockSz, bool slabAligned); + bool coalescAndPutList(FreeBlock *head, bool forceCoalescQDrop, bool reportBlocksProcessed); + + // Main coalescing operation + FreeBlock *doCoalesc(FreeBlock *fBlock, MemRegion **memRegion); + + // Queue for conflicted blocks during coalescing + bool scanCoalescQ(bool forceCoalescQDrop); + intptr_t blocksInCoalescing() const { return coalescQ.blocksInFly(); } + + /*--------------------- FreeBlock backend accessors ---------------------------*/ + FreeBlock *genericGetBlock(int num, size_t size, bool slabAligned); + void genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned); + + // Split the block and return remaining parts to backend if possible + FreeBlock *splitBlock(FreeBlock *fBlock, int num, size_t size, bool isAligned, bool needAlignedBlock); + + void removeBlockFromBin(FreeBlock *fBlock); + + // TODO: combine with returnLargeObject + void putLargeBlock(LargeMemoryBlock *lmb); + + /*------------------- Starting point for OS allocation ------------------------*/ + void requestBootstrapMem(); + FreeBlock *askMemFromOS(size_t totalReqSize, intptr_t startModifiedCnt, + int *lockedBinsThreshold, int numOfLockedBins, + bool *splittable, bool needSlabRegion); + + /*---------------------- Memory regions allocation ----------------------------*/ + FreeBlock *addNewRegion(size_t size, MemRegionType type, bool addToBin); + void releaseRegion(MemRegion *region); + + // TODO: combine in one initMemoryRegion function + FreeBlock *findBlockInRegion(MemRegion *region, size_t exactBlockSize); + void startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin); + + /*------------------------- Raw memory accessors ------------------------------*/ + void *allocRawMem(size_t &size); + bool freeRawMem(void *object, size_t size); + + /*------------------------------ Cleanup functions ----------------------------*/ + // Clean all memory from all caches (extMemPool hard cleanup) + FreeBlock *releaseMemInCaches(intptr_t startModifiedCnt, int *lockedBinsThreshold, int numOfLockedBins); + // Soft heap limit (regular cleanup, then maybe hard cleanup) + void releaseCachesToLimit(); + + /*---------------------------------- Utility ----------------------------------*/ + // TODO: move inside IndexedBins class + static int sizeToBin(size_t size) { + if (size >= maxBinned_HugePage) + return HUGE_BIN; + else if (size < minBinnedSize) + return NO_BIN; + + int bin = (size - minBinnedSize)/freeBinsStep; + + MALLOC_ASSERT(bin < HUGE_BIN, "Invalid size."); + return bin; + } + static bool toAlignedBin(FreeBlock *block, size_t size) { + return isAligned((char*)block + size, slabSize) && size >= slabSize; + } + +public: + /*--------------------- Init, reset, destroy, verify -------------------------*/ + void init(ExtMemoryPool *extMemoryPool); + bool destroy(); + + void verify(); + void reset(); + bool clean(); // clean on caches cleanup + + /*------------------------- Slab block request --------------------------------*/ + BlockI *getSlabBlock(int num); + void putSlabBlock(BlockI *block); + + /*-------------------------- Large object request -----------------------------*/ + LargeMemoryBlock *getLargeBlock(size_t size); + // TODO: make consistent with getLargeBlock + void returnLargeObject(LargeMemoryBlock *lmb); + + /*-------------------------- Backreference memory request ----------------------*/ + void *getBackRefSpace(size_t size, bool *rawMemUsed); + void putBackRefSpace(void *b, size_t size, bool rawMemUsed); + + /*----------------------------- Remap object ----------------------------------*/ + void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment); + + /*---------------------------- Validation -------------------------------------*/ + bool inUserPool() const; + bool ptrCanBeValid(void *ptr) const { return usedAddrRange.inRange(ptr); } + + /*-------------------------- Configuration API --------------------------------*/ + // Soft heap limit + void setRecommendedMaxSize(size_t softLimit) { + memSoftLimit = softLimit; + releaseCachesToLimit(); + } + + /*------------------------------- Info ----------------------------------------*/ + size_t getMaxBinnedSize() const; + + /*-------------------------- Testing, statistics ------------------------------*/ +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getTotalMemSize() const { return totalMemSize; } +#endif +#if __TBB_MALLOC_BACKEND_STAT + void reportStat(FILE *f); +private: + static size_t binToSize(int bin) { + MALLOC_ASSERT(bin <= HUGE_BIN, "Invalid bin."); + + return bin*freeBinsStep + minBinnedSize; + } +#endif +}; + +#endif // __TBB_backend_H diff --git a/src/tbb/src/tbbmalloc/backref.cpp b/src/tbb/src/tbbmalloc/backref.cpp index 3343e5b7..9016d58a 100644 --- a/src/tbb/src/tbbmalloc/backref.cpp +++ b/src/tbb/src/tbbmalloc/backref.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbbmalloc_internal.h" @@ -159,7 +155,7 @@ void BackRefMaster::initEmptyBackRefBlock(BackRefBlock *newBl) bool BackRefMaster::requestNewSpace() { - bool rawMemUsed; + bool isRawMemUsed; MALLOC_STATIC_ASSERT(!(blockSpaceSize % BackRefBlock::bytes), "Must request space for whole number of blocks."); @@ -172,7 +168,7 @@ bool BackRefMaster::requestNewSpace() if (listForUse) // double check that only one block is available return true; BackRefBlock *newBl = - (BackRefBlock*)backend->getBackRefSpace(blockSpaceSize, &rawMemUsed); + (BackRefBlock*)backend->getBackRefSpace(blockSpaceSize, &isRawMemUsed); if (!newBl) return false; // touch a page for the 1st time without taking masterMutex ... @@ -184,7 +180,7 @@ bool BackRefMaster::requestNewSpace() const size_t numOfUnusedIdxs = backRefMaster->dataSz - lastUsed - 1; if (numOfUnusedIdxs <= 0) { // no space in master under lock, roll back - backend->putBackRefSpace(newBl, blockSpaceSize, rawMemUsed); + backend->putBackRefSpace(newBl, blockSpaceSize, isRawMemUsed); return false; } // It's possible that only part of newBl is used, due to lack of indices in master. @@ -193,7 +189,7 @@ bool BackRefMaster::requestNewSpace() // use the first block in the batch to maintain the list of "raw" memory // to be released at shutdown - if (rawMemUsed) { + if (isRawMemUsed) { newBl->nextRawMemBlock = backRefMaster->allRawMemBlocks; backRefMaster->allRawMemBlocks = newBl; } diff --git a/src/tbb/src/tbbmalloc/frontend.cpp b/src/tbb/src/tbbmalloc/frontend.cpp index 9aa9a135..d780437f 100644 --- a/src/tbb/src/tbbmalloc/frontend.cpp +++ b/src/tbb/src/tbbmalloc/frontend.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ @@ -25,6 +21,7 @@ #include /* for memset */ #include "../tbb/tbb_version.h" +#include "../tbb/tbb_environment.h" #include "../tbb/itt_notify.h" // for __TBB_load_ittnotify() #if USE_PTHREAD @@ -139,16 +136,19 @@ class ThreadId { return result; } public: - static void init() { + static bool init() { #if USE_WINTHREAD Tid_key = TlsAlloc(); + if (Tid_key == TLS_ALLOC_FAILURE) + return false; #else int status = pthread_key_create( &Tid_key, NULL ); if ( status ) { - fprintf (stderr, "The memory manager cannot create tls key during initialization; exiting \n"); - exit(1); + fprintf (stderr, "The memory manager cannot create tls key during initialization\n"); + return false; } #endif /* USE_WINTHREAD */ + return true; } static void destroy() { if( Tid_key ) { @@ -157,10 +157,8 @@ class ThreadId { #else int status = pthread_key_delete( Tid_key ); #endif /* USE_WINTHREAD */ - if ( status ) { - fprintf (stderr, "The memory manager cannot delete tls key; exiting \n"); - exit(1); - } + if ( status ) + fprintf (stderr, "The memory manager cannot delete tls key\n"); Tid_key = 0; } } @@ -203,7 +201,7 @@ class ThreadId { #else bool isCurrentThreadId() const { return GetCurrentThreadId() == tid; } #endif - static void init() {} + static bool init() { return true; } static void destroy() {} }; @@ -277,11 +275,12 @@ class MemoryPool { ExtMemoryPool extMemPool; BootStrapBlocks bootStrapBlocks; - bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy); static void initDefaultPool(); + + bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy); bool reset(); bool destroy(); - void processThreadShutdown(TLSData *tlsData); + void onThreadShutdown(TLSData *tlsData); inline TLSData *getTLS(bool create); void clearTLS() { extMemPool.tlsPointerKey.setThreadMallocTLS(NULL); } @@ -355,17 +354,25 @@ class LocalBlockFields : public GlobalBlockFields, Padding { public: - bool empty() const { return allocatedCount==0 && !isSolidPtr(publicFreeList); } + bool empty() const { + if (allocatedCount > 0) return false; + MALLOC_ASSERT(!isSolidPtr(publicFreeList), ASSERT_TEXT); + return true; + } inline FreeObject* allocate(); inline FreeObject *allocateFromFreeList(); - inline bool emptyEnoughToUse(); + + inline bool adjustFullness(); + void adjustPositionInBin(Bin* bin = NULL); + bool freeListNonNull() { return freeList; } void freePublicObject(FreeObject *objectToFree); inline void freeOwnObject(void *object); void reset(); - void privatizePublicFreeList( bool cleanup = false ); + void privatizePublicFreeList( bool reset = true ); void restoreBumpPtr(); void privatizeOrphaned(TLSData *tls, unsigned index); + bool readyToShare(); void shareOrphaned(intptr_t binTag, unsigned index); unsigned int getSize() const { MALLOC_ASSERT(isStartupAllocObject() || objectSizeallocatedCount; } inline void setActiveBlock(Block *block); inline Block* setPreviousBlockActive(); - Block* getPublicFreeListBlock(); + Block* getPrivatizedFreeListBlock(); void moveBlockToFront(Block *block); - void processLessUsedBlock(MemoryPool *memPool, Block *block); + bool cleanPublicFreeLists(); + void processEmptyBlock(Block *block, bool poolTheBlock); + void addPublicFreeListBlock(Block* block); void outofTLSBin(Block* block); void verifyTLSBin(size_t size) const; void pushTLSBin(Block* block); void verifyInitState() const { - MALLOC_ASSERT( activeBlk == 0, ASSERT_TEXT ); - MALLOC_ASSERT( mailbox == 0, ASSERT_TEXT ); + MALLOC_ASSERT( !activeBlk, ASSERT_TEXT ); + MALLOC_ASSERT( !mailbox, ASSERT_TEXT ); } friend void Block::freePublicObject (FreeObject *objectToFree); @@ -518,6 +528,7 @@ const uint32_t minLargeObjectSize = fittingSize5 + 1; * threads memory that are likely in local cache(s) of our CPU. */ class FreeBlockPool { +private: Block *head; int size; Backend *backend; @@ -543,6 +554,7 @@ class FreeBlockPool { template class LocalLOCImpl { +private: static const size_t MAX_TOTAL_SIZE = 4*1024*1024; // TODO: can single-linked list be faster here? LargeMemoryBlock *head, @@ -577,13 +589,16 @@ class TLSData : public TLSRemote { TLSData(MemoryPool *mPool, Backend *bknd) : memPool(mPool), freeSlabBlocks(bknd) {} MemoryPool *getMemPool() const { return memPool; } Bin* getAllocationBin(size_t size); - void release(MemoryPool *mPool); - bool externalCleanup(ExtMemoryPool *mPool, bool cleanOnlyUnused) { + void release(); + bool externalCleanup(bool cleanOnlyUnused, bool cleanBins) { if (!unused && cleanOnlyUnused) return false; + // Heavy operation in terms of synchronization complexity, + // should be called only for the current thread + bool released = cleanBins ? cleanupBlockBins() : false; // both cleanups to be called, and the order is not important - return lloc.externalCleanup(mPool) | freeSlabBlocks.externalCleanup(); + return released | lloc.externalCleanup(&memPool->extMemPool) | freeSlabBlocks.externalCleanup(); } - bool cleanUnusedActiveBlocks(Backend *backend, bool userPool); + bool cleanupBlockBins(); void markUsed() { unused = false; } // called by owner when TLS touched void markUnused() { unused = true; } // can be called by not owner thread }; @@ -605,31 +620,31 @@ TLSData *TLSKey::createTLS(MemoryPool *memPool, Backend *backend) return tls; } -bool TLSData::cleanUnusedActiveBlocks(Backend *backend, bool userPool) +bool TLSData::cleanupBlockBins() { bool released = false; - // active blocks can be not used, so return them to backend - for (uint32_t i=0; iempty()) { bin[i].outofTLSBin(block); - // slab blocks in user's pools do not have valid backRefIdx - if (!userPool) - removeBackRef(*(block->getBackRefIdx())); - backend->putSlabBlock(block); - + memPool->returnEmptyBlock(block, /*poolTheBlock=*/false); released = true; } + } return released; } bool ExtMemoryPool::releaseAllLocalCaches() { - bool released = allLocalCaches.cleanup(this, /*cleanOnlyUnused=*/false); + // Iterate all registered TLS data and clean LLOC and Slab pools + bool released = allLocalCaches.cleanup(/*cleanOnlyUnused=*/false); + // Bins privatization is done only for the current thread if (TLSData *tlsData = tlsPointerKey.getThreadMallocTLS()) - // released only for current thread for now - released |= tlsData->cleanUnusedActiveBlocks(&backend, userPool()); + released |= tlsData->cleanupBlockBins(); return released; } @@ -659,17 +674,15 @@ void AllLocalCaches::unregisterThread(TLSRemote *tls) MALLOC_ASSERT(!tls->next || tls->next->next!=tls->next, ASSERT_TEXT); } -bool AllLocalCaches::cleanup(ExtMemoryPool *extPool, bool cleanOnlyUnused) +bool AllLocalCaches::cleanup(bool cleanOnlyUnused) { - bool total = false; + bool released = false; { MallocMutex::scoped_lock lock(listLock); - for (TLSRemote *curr=head; curr; curr=curr->next) - total |= static_cast(curr)-> - externalCleanup(extPool, cleanOnlyUnused); + released |= static_cast(curr)->externalCleanup(cleanOnlyUnused, /*cleanBins=*/false); } - return total; + return released; } void AllLocalCaches::markUnused() @@ -677,7 +690,7 @@ void AllLocalCaches::markUnused() bool locked; MallocMutex::scoped_lock lock(listLock, /*block=*/false, &locked); if (!locked) // not wait for marking if someone doing something with it - return; + return; for (TLSRemote *curr=head; curr; curr=curr->next) static_cast(curr)->markUnused(); @@ -747,7 +760,7 @@ static inline unsigned int highestBitPos(unsigned int n) unsigned int pos; #if __ARCH_x86_32||__ARCH_x86_64 -# if __linux__||__APPLE__||__FreeBSD__||__NetBSD__||__sun||__MINGW32__ +# if __linux__||__APPLE__||__FreeBSD__||__NetBSD__||__OpenBSD__||__sun||__MINGW32__ __asm__ ("bsr %1,%0" : "=r"(pos) : "r"(n)); # elif (_WIN32 && (!_WIN64 || __INTEL_COMPILER)) __asm @@ -980,7 +993,7 @@ inline Bin* TLSData::getAllocationBin(size_t size) /* Return an empty uninitialized block in a non-blocking fashion. */ Block *MemoryPool::getEmptyBlock(size_t size) { - TLSData* tls = extMemPool.tlsPointerKey.getThreadMallocTLS(); + TLSData* tls = getTLS(/*create=*/false); // try to use per-thread cache, if TLS available FreeBlockPool::ResOfGet resOfGet = tls? tls->freeSlabBlocks.getBlock() : FreeBlockPool::ResOfGet(NULL, false); @@ -1035,9 +1048,8 @@ void MemoryPool::returnEmptyBlock(Block *block, bool poolTheBlock) { block->reset(); if (poolTheBlock) { - extMemPool.tlsPointerKey.getThreadMallocTLS()->freeSlabBlocks.returnBlock(block); - } - else { + getTLS(/*create=*/false)->freeSlabBlocks.returnBlock(block); + } else { // slab blocks in user's pools do not have valid backRefIdx if (!extMemPool.userPool()) removeBackRef(*(block->getBackRefIdx())); @@ -1056,7 +1068,7 @@ bool ExtMemoryPool::init(intptr_t poolId, rawAllocType rawAlloc, this->keepAllMemory = keepAllMemory; this->fixedPool = fixedPool; this->delayRegsReleasing = false; - if (! initTLS()) + if (!initTLS()) return false; loc.init(this); backend.init(this); @@ -1133,11 +1145,13 @@ bool MemoryPool::destroy() return extMemPool.destroy(); } -void MemoryPool::processThreadShutdown(TLSData *tlsData) +void MemoryPool::onThreadShutdown(TLSData *tlsData) { - tlsData->release(this); - bootStrapBlocks.free(tlsData); - clearTLS(); + if (tlsData) { // might be called for "empty" TLS + tlsData->release(); + bootStrapBlocks.free(tlsData); + clearTLS(); + } } #if MALLOC_DEBUG @@ -1221,7 +1235,7 @@ void Bin::outofTLSBin(Block* block) if (block == activeBlk) { activeBlk = block->previous? block->previous : block->next; } - /* Delink the block */ + /* Unlink the block */ if (block->previous) { MALLOC_ASSERT( block->previous->next == block, ASSERT_TEXT ); block->previous->next = block->next; @@ -1236,7 +1250,7 @@ void Bin::outofTLSBin(Block* block) verifyTLSBin(size); } -Block* Bin::getPublicFreeListBlock() +Block* Bin::getPrivatizedFreeListBlock() { Block* block; MALLOC_ASSERT( this, ASSERT_TEXT ); @@ -1259,42 +1273,90 @@ Block* Bin::getPublicFreeListBlock() if( block ) { MALLOC_ASSERT( isSolidPtr(block->publicFreeList), ASSERT_TEXT ); block->privatizePublicFreeList(); + block->adjustPositionInBin(this); } return block; } -bool Block::emptyEnoughToUse() +void Bin::addPublicFreeListBlock(Block* block) +{ + MallocMutex::scoped_lock scoped_cs(mailLock); + block->nextPrivatizable = mailbox; + mailbox = block; +} + +// Process publicly freed objects in all blocks and return empty blocks +// to the backend in order to reduce overall footprint. +bool Bin::cleanPublicFreeLists() { - const float threshold = (slabSize - sizeof(Block)) * (1-emptyEnoughRatio); + Block* block; + if (!FencedLoad((intptr_t&)mailbox)) + return false; + else { + // Grab all the blocks in the mailbox + MallocMutex::scoped_lock scoped_cs(mailLock); + block = mailbox; + mailbox = NULL; + } + bool released = false; + while (block) { + MALLOC_ASSERT( block->isOwnedByCurrentThread(), ASSERT_TEXT ); + Block* tmp = block->nextPrivatizable; + block->nextPrivatizable = (Block*) this; + block->privatizePublicFreeList(); + if (block->empty()) { + processEmptyBlock(block, /*poolTheBlock=*/false); + released = true; + } else + block->adjustPositionInBin(this); + block = tmp; + } + return released; +} +bool Block::adjustFullness() +{ if (bumpPtr) { /* If we are still using a bump ptr for this block it is empty enough to use. */ STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); isFull = false; - return 1; - } - - /* allocatedCount shows how many objects in the block are in use; however it still counts - blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ - isFull = (allocatedCount*objectSize > threshold)? true: false; + } else { + const float threshold = (slabSize - sizeof(Block)) * (1 - emptyEnoughRatio); + /* allocatedCount shows how many objects in the block are in use; however it still counts + * blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ + isFull = (allocatedCount*objectSize > threshold) ? true : false; #if COLLECT_STATISTICS - if (isFull) - STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); - else - STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); + if (isFull) + STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); + else + STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); #endif - return !isFull; + } + return isFull; +} + +// This method resides in class Block, and not in class Bin, in order to avoid +// calling getAllocationBin on a reasonably hot path in Block::freeOwnObject +void Block::adjustPositionInBin(Bin* bin/*=NULL*/) +{ + // If the block were full, but became empty enough to use, + // move it to the front of the list + if (isFull && !adjustFullness()) { + if (!bin) + bin = tlsPtr->getAllocationBin(objectSize); + bin->moveBlockToFront(this); + } } /* Restore the bump pointer for an empty block that is planned to use */ void Block::restoreBumpPtr() { MALLOC_ASSERT( allocatedCount == 0, ASSERT_TEXT ); - MALLOC_ASSERT( publicFreeList == NULL, ASSERT_TEXT ); + MALLOC_ASSERT( !isSolidPtr(publicFreeList), ASSERT_TEXT ); STAT_increment(getThreadId(), getIndex(objectSize), freeRestoreBumpPtr); bumpPtr = (FreeObject *)((uintptr_t)this + slabSize - objectSize); freeList = NULL; - isFull = 0; + isFull = false; } void Block::freeOwnObject(void *object) @@ -1310,19 +1372,14 @@ void Block::freeOwnObject(void *object) STAT_increment(getThreadId(), getIndex(objectSize), freeToActiveBlock); #endif if (empty()) { - // The bump pointer is about to be restored for the block, - // no need to find objectToFree here (this is costly). - - // if the last object of a slab is freed, the slab cannot be marked full + // If the last object of a slab is freed, the slab cannot be marked full MALLOC_ASSERT(!isFull, ASSERT_TEXT); - tlsPtr->getAllocationBin(objectSize)->processLessUsedBlock(poolPtr, this); - } else { + tlsPtr->getAllocationBin(objectSize)->processEmptyBlock(this, /*poolTheBlock=*/true); + } else { // hot path FreeObject *objectToFree = findObjectToFree(object); objectToFree->next = freeList; freeList = objectToFree; - - if (isFull && emptyEnoughToUse()) - tlsPtr->getAllocationBin(objectSize)->moveBlockToFront(this); + adjustPositionInBin(); } } @@ -1360,30 +1417,25 @@ void Block::freePublicObject (FreeObject *objectToFree) if( !isNotForUse(nextPrivatizable) ) { MALLOC_ASSERT( nextPrivatizable!=NULL, ASSERT_TEXT ); Bin* theBin = (Bin*) nextPrivatizable; - MallocMutex::scoped_lock scoped_cs(theBin->mailLock); - nextPrivatizable = theBin->mailbox; - theBin->mailbox = this; + theBin->addPublicFreeListBlock(this); } } STAT_increment(getThreadId(), ThreadCommonCounters, freeToOtherThread); STAT_increment(ownerTid, getIndex(objectSize), freeByOtherThread); } -void Block::privatizePublicFreeList( bool cleanup ) +// Make objects freed by other threads available for use again +void Block::privatizePublicFreeList( bool reset ) { - FreeObject *temp, *localPublicFreeList; - const intptr_t endMarker = cleanup? UNUSABLE : 0; + FreeObject *localPublicFreeList; + // If reset is false, publicFreeList should not be zeroed but set to UNUSABLE + // to properly synchronize with other threads freeing objects to this slab. + const intptr_t endMarker = reset ? 0 : UNUSABLE; - // During cleanup of orphaned blocks, the calling thread is not registered as the owner - MALLOC_ASSERT( cleanup || isOwnedByCurrentThread(), ASSERT_TEXT ); + // Only the owner thread may reset the pointer to NULL + MALLOC_ASSERT( isOwnedByCurrentThread() || !reset, ASSERT_TEXT ); #if FREELIST_NONBLOCKING - temp = publicFreeList; - do { - localPublicFreeList = temp; - temp = (FreeObject*)AtomicCompareExchange( (intptr_t&)publicFreeList, - endMarker, (intptr_t)localPublicFreeList); - // no backoff necessary because trying to make change, not waiting for a change - } while( temp != localPublicFreeList ); + localPublicFreeList = (FreeObject*)AtomicFetchStore( &publicFreeList, endMarker ); #else STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); { @@ -1391,17 +1443,17 @@ void Block::privatizePublicFreeList( bool cleanup ) localPublicFreeList = publicFreeList; publicFreeList = endMarker; } - temp = localPublicFreeList; #endif MALLOC_ITT_SYNC_ACQUIRED(&publicFreeList); + MALLOC_ASSERT( !(reset && isNotForUse(publicFreeList)), ASSERT_TEXT ); - // publicFreeList must have been UNUSABLE (possible for orphaned blocks) or valid, but not NULL + // publicFreeList must have been UNUSABLE or valid, but not NULL MALLOC_ASSERT( localPublicFreeList!=NULL, ASSERT_TEXT ); - MALLOC_ASSERT( localPublicFreeList==temp, ASSERT_TEXT ); - if( isSolidPtr(temp) ) { + if( isSolidPtr(localPublicFreeList) ) { MALLOC_ASSERT( allocatedCount <= (slabSize-sizeof(Block))/objectSize, ASSERT_TEXT ); /* other threads did not change the counter freeing our blocks */ allocatedCount--; + FreeObject *temp = localPublicFreeList; while( isSolidPtr(temp->next) ){ // the list will end with either NULL or UNUSABLE temp = temp->next; allocatedCount--; @@ -1428,34 +1480,41 @@ void Block::privatizeOrphaned(TLSData *tls, unsigned index) nextPrivatizable = (Block*)bin; // the next call is required to change publicFreeList to 0 privatizePublicFreeList(); - if( allocatedCount ) { - emptyEnoughToUse(); // check its fullness and set result->isFull - } else { + if( empty() ) { restoreBumpPtr(); + } else { + adjustFullness(); // check the block fullness and set isFull } MALLOC_ASSERT( !isNotForUse(publicFreeList), ASSERT_TEXT ); } + +bool Block::readyToShare() +{ + void* oldval; +#if FREELIST_NONBLOCKING + oldval = (void*)AtomicCompareExchange((intptr_t&)publicFreeList, UNUSABLE, 0); +#else + STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); + { + MallocMutex::scoped_lock scoped_cs(publicFreeListLock); + if ( (oldval=publicFreeList)==NULL ) + (intptr_t&)(publicFreeList) = UNUSABLE; + } +#endif + return oldval==NULL; +} + void Block::shareOrphaned(intptr_t binTag, unsigned index) { MALLOC_ASSERT( binTag, ASSERT_TEXT ); STAT_increment(getThreadId(), index, freeBlockPublic); markOrphaned(); - // need to set publicFreeList to non-zero, so other threads - // will not change nextPrivatizable and it can be zeroed. if ((intptr_t)nextPrivatizable==binTag) { - void* oldval; -#if FREELIST_NONBLOCKING - oldval = (void*)AtomicCompareExchange((intptr_t&)publicFreeList, UNUSABLE, 0); -#else - STAT_increment(getThreadId(), ThreadCommonCounters, lockPublicFreeList); - { - MallocMutex::scoped_lock scoped_cs(publicFreeListLock); - if ( (oldval=publicFreeList)==NULL ) - (intptr_t&)(publicFreeList) = UNUSABLE; - } -#endif - if ( oldval!=NULL ) { + // First check passed: the block is not in mailbox yet. + // Need to set publicFreeList to non-zero, so other threads + // will not change nextPrivatizable and it can be zeroed. + if ( !readyToShare() ) { // another thread freed an object; we need to wait until it finishes. // There is no need for exponential backoff, as the wait here is not for a lock; // but need to yield, so the thread we wait has a chance to run. @@ -1468,8 +1527,6 @@ void Block::shareOrphaned(intptr_t binTag, unsigned index) } } } - } else { - MALLOC_ASSERT( isSolidPtr(publicFreeList), ASSERT_TEXT ); } MALLOC_ASSERT( publicFreeList!=NULL, ASSERT_TEXT ); // now it is safe to change our data @@ -1486,7 +1543,7 @@ void Block::cleanBlockHeader() previous = NULL; freeList = NULL; allocatedCount = 0; - isFull = 0; + isFull = false; tlsPtr = NULL; publicFreeList = NULL; @@ -1540,20 +1597,20 @@ void OrphanedBlocks::reset() bool OrphanedBlocks::cleanup(Backend* backend) { - bool result = false; + bool released = false; for (uint32_t i=0; inext; - block->privatizePublicFreeList( /*cleanup=*/true ); + block->privatizePublicFreeList( /*reset=*/false ); // do not set publicFreeList to NULL if (block->empty()) { block->reset(); // slab blocks in user's pools do not have valid backRefIdx if (!backend->inUserPool()) removeBackRef(*(block->getBackRefIdx())); backend->putSlabBlock(block); - result = true; + released = true; } else { MALLOC_ITT_SYNC_RELEASING(bins+i); bins[i].push(block); @@ -1561,7 +1618,7 @@ bool OrphanedBlocks::cleanup(Backend* backend) block = next; } } - return result; + return released; } FreeBlockPool::ResOfGet FreeBlockPool::getBlock() @@ -1612,7 +1669,7 @@ void FreeBlockPool::returnBlock(Block *block) bool FreeBlockPool::externalCleanup() { Block *helper; - bool nonEmpty = false; + bool released = false; for (Block *currBl=(Block*)AtomicFetchStore(&head, 0); currBl; currBl=helper) { helper = currBl->next; @@ -1620,9 +1677,9 @@ bool FreeBlockPool::externalCleanup() if (!backend->inUserPool()) removeBackRef(currBl->backRefIdx); backend->putSlabBlock(currBl); - nonEmpty = true; + released = true; } - return nonEmpty; + return released; } /* Prepare the block for returning to FreeBlockPool */ @@ -1656,7 +1713,7 @@ inline Block* Bin::setPreviousBlockActive() MALLOC_ASSERT( activeBlk, ASSERT_TEXT ); Block* temp = activeBlk->previous; if( temp ) { - MALLOC_ASSERT( temp->isFull == 0, ASSERT_TEXT ); + MALLOC_ASSERT( !(temp->isFull), ASSERT_TEXT ); activeBlk = temp; } return temp; @@ -1691,10 +1748,10 @@ FreeObject *Block::findObjectToFree(const void *object) const return objectToFree; } -void TLSData::release(MemoryPool *mPool) +void TLSData::release() { - mPool->extMemPool.allLocalCaches.unregisterThread(this); - externalCleanup(&mPool->extMemPool, /*cleanOnlyUnused=*/false); + memPool->extMemPool.allLocalCaches.unregisterThread(this); + externalCleanup(/*cleanOnlyUnused=*/false, /*cleanBins=*/false); for (unsigned index = 0; index < numBlockBins; index++) { Block *activeBlk = bin[index].getActiveBlock(); @@ -1705,9 +1762,9 @@ void TLSData::release(MemoryPool *mPool) Block *threadBlock = threadlessBlock->previous; if (threadlessBlock->empty()) { /* we destroy the thread, so not use its block pool */ - mPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); + memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); } else { - mPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); } threadlessBlock = threadBlock; } @@ -1716,9 +1773,9 @@ void TLSData::release(MemoryPool *mPool) Block *threadBlock = threadlessBlock->next; if (threadlessBlock->empty()) { /* we destroy the thread, so not use its block pool */ - mPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); + memPool->returnEmptyBlock(threadlessBlock, /*poolTheBlock=*/false); } else { - mPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); + memPool->extMemPool.orphanedBlocks.put(intptr_t(bin+index), threadlessBlock); } threadlessBlock = threadBlock; } @@ -1873,54 +1930,6 @@ static MallocMutex initMutex; delivers a clean result. */ static char VersionString[] = "\0" TBBMALLOC_VERSION_STRINGS; -#if __TBB_WIN8UI_SUPPORT -bool GetBoolEnvironmentVariable(const char *) { return false; } -#else -bool GetBoolEnvironmentVariable(const char *name) -{ - if (const char* s = getenv(name)) - return strcmp(s,"0") != 0; - return false; -} -#endif - -void AllocControlledMode::initReadEnv(const char *envName, intptr_t defaultVal) -{ - if (!setDone) { -#if !__TBB_WIN8UI_SUPPORT - // TODO: use strtol to get the actual value of the envirable - const char *envVal = getenv(envName); - if (envVal && !strcmp(envVal, "1")) - val = 1; - else -#endif - val = defaultVal; - setDone = true; - } -} - -void MemoryPool::initDefaultPool() -{ - long long unsigned hugePageSize = 0; -#if __linux__ - if (FILE *f = fopen("/proc/meminfo", "r")) { - const int READ_BUF_SIZE = 100; - char buf[READ_BUF_SIZE]; - MALLOC_STATIC_ASSERT(sizeof(hugePageSize) >= 8, - "At least 64 bits required for keeping page size/numbers."); - - while (fgets(buf, READ_BUF_SIZE, f)) { - if (1 == sscanf(buf, "Hugepagesize: %llu kB", &hugePageSize)) { - hugePageSize *= 1024; - break; - } - } - fclose(f); - } -#endif - hugePages.init(hugePageSize); -} - #if USE_PTHREAD && (__TBB_SOURCE_DIRECTLY_INCLUDED || __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND) /* Decrease race interval between dynamic library unloading and pthread key @@ -1933,8 +1942,8 @@ class ShutdownSync { static const intptr_t skipDtor = INTPTR_MIN/2; public: void init() { flag = 0; } -/* Suppose that 2*abs(skipDtor) or more threads never call threadExitStart() - simultaneously, so flag is never becomes negative because of that. */ +/* Suppose that 2*abs(skipDtor) or more threads never call threadDtorStart() + simultaneously, so flag never becomes negative because of that. */ bool threadDtorStart() { if (flag < 0) return false; @@ -1977,7 +1986,7 @@ bool isMallocInitializedExt() { return isMallocInitialized(); } -/** Caller is responsible for ensuring this routine is called exactly once. */ +/* Caller is responsible for ensuring this routine is called exactly once. */ extern "C" void MallocInitializeITT() { #if DO_ITT_NOTIFY if (!usedBySrcIncluded) @@ -1985,6 +1994,10 @@ extern "C" void MallocInitializeITT() { #endif } +void MemoryPool::initDefaultPool() { + hugePages.init(); +} + /* * Allocator initialization routine; * it is called lazily on the very first scalable_malloc call. @@ -2004,13 +2017,17 @@ static bool initMemoryManager() // POSIX.1-2001-compliant way to get page size const size_t granularity = sysconf(_SC_PAGESIZE); #endif + if (!defaultMemPool) { + // Do not rely on static constructors and do the assignment in case + // of library static section not initialized at this call yet. + defaultMemPool = (MemoryPool*)defaultMemPool_space; + } bool initOk = defaultMemPool-> extMemPool.init(0, NULL, NULL, granularity, /*keepAllMemory=*/false, /*fixedPool=*/false); // TODO: extMemPool.init() to not allocate memory - if (!initOk || !initBackRefMaster(&defaultMemPool->extMemPool.backend)) + if (!initOk || !initBackRefMaster(&defaultMemPool->extMemPool.backend) || !ThreadId::init()) return false; - ThreadId::init(); // Create keys for thread id MemoryPool::initDefaultPool(); // init() is required iff initMemoryManager() is called // after mallocProcessShutdownNotification() @@ -2021,6 +2038,10 @@ static bool initMemoryManager() return true; } +static bool GetBoolEnvironmentVariable(const char* name) { + return tbb::internal::GetBoolEnvironmentVariable(name); +} + //! Ensures that initMemoryManager() is called once and only once. /** Does not return until initMemoryManager() has been completed by a thread. There is no need to call this routine if mallocInitialized==2 . */ @@ -2110,7 +2131,7 @@ inline FreeObject* Block::allocate() MALLOC_ASSERT( !bumpPtr, ASSERT_TEXT ); /* the block is considered full. */ - isFull = 1; + isFull = true; return NULL; } @@ -2139,12 +2160,12 @@ void Bin::moveBlockToFront(Block *block) pushTLSBin(block); } -void Bin::processLessUsedBlock(MemoryPool *memPool, Block *block) +void Bin::processEmptyBlock(Block *block, bool poolTheBlock) { if (block != activeBlk) { - /* We are not actively using this block; return it to the general block pool */ + /* We are not using this block; return it to the pool */ outofTLSBin(block); - memPool->returnEmptyBlock(block, /*poolTheBlock=*/true); + block->getMemPool()->returnEmptyBlock(block, poolTheBlock); } else { /* all objects are free - let's restore the bump pointer */ block->restoreBumpPtr(); @@ -2199,7 +2220,7 @@ LargeMemoryBlock *LocalLOCImpl::get(size_t size) if (size > MAX_TOTAL_SIZE) return NULL; - if (!head || !(localHead = (LargeMemoryBlock*)AtomicFetchStore(&head, 0))) { + if (!head || (localHead = (LargeMemoryBlock*)AtomicFetchStore(&head, 0)) == NULL) { // do not restore totalSize, numOfBlocks and tail at this point, // as they are used only in put(), where they must be restored return NULL; @@ -2355,7 +2376,7 @@ static void *allocateAligned(MemoryPool *memPool, size_t size, size_t alignment) } static void *reallocAligned(MemoryPool *memPool, void *ptr, - size_t size, size_t alignment = 0) + size_t newSize, size_t alignment = 0) { void *result; size_t copySize; @@ -2363,32 +2384,46 @@ static void *reallocAligned(MemoryPool *memPool, void *ptr, if (isLargeObject(ptr)) { LargeMemoryBlock* lmb = ((LargeObjectHdr *)ptr - 1)->memoryBlock; copySize = lmb->unalignedSize-((uintptr_t)ptr-(uintptr_t)lmb); - if (size <= copySize && (0==alignment || isAligned(ptr, alignment))) { - lmb->objectSize = size; - return ptr; - } else { - copySize = lmb->objectSize; + + // Apply different strategies if size decreases + if (newSize <= copySize && (0 == alignment || isAligned(ptr, alignment))) { + + // For huge objects (that do not fit in backend cache), keep the same space unless + // the new size is at least twice smaller + bool isMemoryBlockHuge = copySize > memPool->extMemPool.backend.getMaxBinnedSize(); + size_t threshold = isMemoryBlockHuge ? copySize / 2 : 0; + if (newSize > threshold) { + lmb->objectSize = newSize; + return ptr; + } + // TODO: For large objects suitable for the backend cache, + // split out the excessive part and put it to the backend. + } + // Reallocate for real + copySize = lmb->objectSize; #if BACKEND_HAS_MREMAP - if (void *r = memPool->extMemPool.remap(ptr, copySize, size, - alignmentextMemPool.remap(ptr, copySize, newSize, + alignment < largeObjectAlignment ? largeObjectAlignment : alignment)) + return r; #endif - result = alignment ? allocateAligned(memPool, size, alignment) : - internalPoolMalloc(memPool, size); - } + result = alignment ? allocateAligned(memPool, newSize, alignment) : + internalPoolMalloc(memPool, newSize); + } else { Block* block = (Block *)alignDown(ptr, slabSize); copySize = block->findObjectSize(ptr); - if (size <= copySize && (0==alignment || isAligned(ptr, alignment))) { + + // TODO: Move object to another bin if size decreases and the current bin is "empty enough". + // Currently, in case of size decreasing, old pointer is returned + if (newSize <= copySize && (0==alignment || isAligned(ptr, alignment))) { return ptr; } else { - result = alignment ? allocateAligned(memPool, size, alignment) : - internalPoolMalloc(memPool, size); + result = alignment ? allocateAligned(memPool, newSize, alignment) : + internalPoolMalloc(memPool, newSize); } } if (result) { - memcpy(result, ptr, copySizebackRefIdx) : - header->backRefIdx; + BackRefIdx idx = (memOrigin == unknownMem) ? + safer_dereference(&header->backRefIdx) : header->backRefIdx; return idx.isLargeObject() // in valid LargeObjectHdr memoryBlock is not NULL @@ -2524,11 +2559,8 @@ static void *internalPoolMalloc(MemoryPool* memPool, size_t size) /* * else privatize publicly freed objects in some block and allocate from it */ - mallocBlock = bin->getPublicFreeListBlock(); + mallocBlock = bin->getPrivatizedFreeListBlock(); if (mallocBlock) { - if (mallocBlock->emptyEnoughToUse()) { - bin->moveBlockToFront(mallocBlock); - } MALLOC_ASSERT( mallocBlock->freeListNonNull(), ASSERT_TEXT ); if ( FreeObject *result = mallocBlock->allocateFromFreeList() ) return result; @@ -2614,18 +2646,15 @@ static void internalFree(void *object) static size_t internalMsize(void* ptr) { - if (ptr) { - MALLOC_ASSERT(isRecognized(ptr), "Invalid pointer in scalable_msize detected."); - if (isLargeObject(ptr)) { - LargeMemoryBlock* lmb = ((LargeObjectHdr*)ptr - 1)->memoryBlock; - return lmb->objectSize; - } else - return ((Block*)alignDown(ptr, slabSize))->findObjectSize(ptr); + MALLOC_ASSERT(ptr, "Invalid pointer passed to internalMsize"); + if (isLargeObject(ptr)) { + // TODO: return the maximum memory size, that can be written to this object + LargeMemoryBlock* lmb = ((LargeObjectHdr*)ptr - 1)->memoryBlock; + return lmb->objectSize; + } else { + Block *block = (Block*)alignDown(ptr, slabSize); + return block->findObjectSize(ptr); } - errno = EINVAL; - // Unlike _msize, return 0 in case of parameter error. - // Returning size_t(-1) looks more like the way to troubles. - return 0; } } // namespace internal @@ -2660,8 +2689,10 @@ rml::MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, return UNSUPPORTED_POLICY; } if (!isMallocInitialized()) - if (!doInitialization()) + if (!doInitialization()) { + *pool = NULL; return NO_MEMORY; + } rml::internal::MemoryPool *memPool = (rml::internal::MemoryPool*)internalMalloc((sizeof(rml::internal::MemoryPool))); if (!memPool) { @@ -2758,6 +2789,21 @@ rml::MemoryPool *pool_identify(void *object) return (rml::MemoryPool*)pool; } +size_t pool_msize(rml::MemoryPool *mPool, void* object) +{ + if (object) { + // No assert for object recognition, cause objects allocated from non-default + // memory pool do not participate in range checking and do not have valid backreferences for + // small objects. Instead, check that an object belong to the certain memory pool. + MALLOC_ASSERT_EX(mPool == pool_identify(object), "Object does not belong to the specified pool"); + return internalMsize(object); + } + errno = EINVAL; + // Unlike _msize, return 0 in case of parameter error. + // Returning size_t(-1) looks more like the way to troubles. + return 0; +} + } // namespace rml using namespace rml::internal; @@ -2771,47 +2817,71 @@ static unsigned int threadGoingDownCount = 0; * from the malloc blocks and replace them with a NULL thread id. * * For pthreads, the function is set as a callback in pthread_key_create for TLS bin. - * For non-NULL keys it will be automatically called at thread exit with the key value - * as the argument. + * It will be automatically called at thread exit with the key value as the argument, + * unless that value is NULL. + * For Windows, it is called from DllMain( DLL_THREAD_DETACH ). + * + * However neither of the above is called for the main process thread, so the routine + * also needs to be called during the process shutdown. * - * for Windows, it should be called directly e.g. from DllMain */ -void mallocThreadShutdownNotification(void* arg) +// TODO: Consider making this function part of class MemoryPool. +void doThreadShutdownNotification(TLSData* tls, bool main_thread) { - // Check whether TLS has been initialized - if (!isMallocInitialized()) return; - TRACEF(( "[ScalableMalloc trace] Thread id %d blocks return start %d\n", getThreadId(), threadGoingDownCount++ )); -#if USE_WINTHREAD - suppress_unused_warning(arg); - MallocMutex::scoped_lock lock(MemoryPool::memPoolListLock); - // The routine is called once per thread, need to walk through all pools on Windows - for (MemoryPool *memPool = defaultMemPool; memPool; memPool = memPool->next) - if (TLSData *tls = memPool->getTLS(/*create=*/false)) - memPool->processThreadShutdown(tls); -#else - if (!shutdownSync.threadDtorStart()) return; - // The routine is called for each memPool, gets memPool from TLSData. - TLSData *tls = (TLSData*)arg; - tls->getMemPool()->processThreadShutdown(tls); - shutdownSync.threadDtorDone(); + +#if USE_PTHREAD + if (tls) { + if (!shutdownSync.threadDtorStart()) return; + tls->getMemPool()->onThreadShutdown(tls); + shutdownSync.threadDtorDone(); + } else #endif + { + suppress_unused_warning(tls); // not used on Windows + // The default pool is safe to use at this point: + // on Linux, only the main thread can go here before destroying defaultMemPool; + // on Windows, shutdown is synchronized via loader lock and isMallocInitialized(). + // See also __TBB_mallocProcessShutdownNotification() + defaultMemPool->onThreadShutdown(defaultMemPool->getTLS(/*create=*/false)); + // Take lock to walk through other pools; but waiting might be dangerous at this point + // (e.g. on Windows the main thread might deadlock) + bool locked; + MallocMutex::scoped_lock lock(MemoryPool::memPoolListLock, /*wait=*/!main_thread, &locked); + if (locked) { // the list is safe to process + for (MemoryPool *memPool = defaultMemPool->next; memPool; memPool = memPool->next) + memPool->onThreadShutdown(memPool->getTLS(/*create=*/false)); + } + } TRACEF(( "[ScalableMalloc trace] Thread id %d blocks return end\n", getThreadId() )); } -#if USE_WINTHREAD +#if USE_PTHREAD +void mallocThreadShutdownNotification(void* arg) +{ + // The routine is called for each pool (as TLS dtor) on each thread, except for the main thread + if (!isMallocInitialized()) return; + doThreadShutdownNotification((TLSData*)arg, false); +} +#else extern "C" void __TBB_mallocThreadShutdownNotification() { - mallocThreadShutdownNotification(NULL); + // The routine is called once per thread on Windows + if (!isMallocInitialized()) return; + doThreadShutdownNotification(NULL, false); } #endif -extern "C" void __TBB_mallocProcessShutdownNotification() +extern "C" void __TBB_mallocProcessShutdownNotification(bool windows_process_dying) { if (!isMallocInitialized()) return; + // Don't clean allocator internals if the entire process is exiting + if (!windows_process_dying) { + doThreadShutdownNotification(NULL, /*main_thread=*/true); + } #if __TBB_MALLOC_LOCACHE_STAT printf("cache hit ratio %f, size hit %f\n", 1.*cacheHits/mallocCalls, 1.*memHitKB/memAllocKB); @@ -2857,12 +2927,14 @@ extern "C" void * scalable_malloc(size_t size) return ptr; } -extern "C" void scalable_free (void *object) { +extern "C" void scalable_free(void *object) +{ internalFree(object); } #if MALLOC_ZONE_OVERLOAD_ENABLED -extern "C" void __TBB_malloc_free_definite_size(void *object, size_t size) { +extern "C" void __TBB_malloc_free_definite_size(void *object, size_t size) +{ internalPoolFree(defaultMemPool, object, size); } #endif @@ -3113,7 +3185,14 @@ extern "C" void scalable_aligned_free(void *ptr) */ extern "C" size_t scalable_msize(void* ptr) { - return internalMsize(ptr); + if (ptr) { + MALLOC_ASSERT(isRecognized(ptr), "Invalid pointer in scalable_msize detected."); + return internalMsize(ptr); + } + errno = EINVAL; + // Unlike _msize, return 0 in case of parameter error. + // Returning size_t(-1) looks more like the way to troubles. + return 0; } /* @@ -3184,6 +3263,9 @@ extern "C" int scalable_allocation_mode(int param, intptr_t value) return TBBMALLOC_INVALID_PARAM; } #endif + } else if (param == TBBMALLOC_SET_HUGE_SIZE_THRESHOLD) { + defaultMemPool->extMemPool.loc.setHugeSizeThreshold((size_t)value); + return TBBMALLOC_OK; } return TBBMALLOC_INVALID_PARAM; } @@ -3192,16 +3274,18 @@ extern "C" int scalable_allocation_command(int cmd, void *param) { if (param) return TBBMALLOC_INVALID_PARAM; + + bool released = false; switch(cmd) { case TBBMALLOC_CLEAN_THREAD_BUFFERS: if (TLSData *tls = defaultMemPool->getTLS(/*create=*/false)) - return tls->externalCleanup(&defaultMemPool->extMemPool, - /*cleanOnlyUnused=*/false)? - TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; - return TBBMALLOC_NO_EFFECT; + released = tls->externalCleanup(/*cleanOnlyUnused*/false, /*cleanBins=*/true); + break; case TBBMALLOC_CLEAN_ALL_BUFFERS: - return defaultMemPool->extMemPool.hardCachesCleanup()? - TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; + released = defaultMemPool->extMemPool.hardCachesCleanup(); + break; + default: + return TBBMALLOC_INVALID_PARAM; } - return TBBMALLOC_INVALID_PARAM; + return released ? TBBMALLOC_OK : TBBMALLOC_NO_EFFECT; } diff --git a/src/tbb/src/tbbmalloc/index.html b/src/tbb/src/tbbmalloc/index.html index 19eaf992..b53ea947 100644 --- a/src/tbb/src/tbbmalloc/index.html +++ b/src/tbb/src/tbbmalloc/index.html @@ -7,7 +7,7 @@

Overview

-Copyright © 2005-2017 Intel Corporation. All Rights Reserved. +Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel is a registered trademark or trademark of Intel Corporation or its subsidiaries in the United States and other countries. diff --git a/src/tbb/src/tbbmalloc/large_objects.cpp b/src/tbb/src/tbbmalloc/large_objects.cpp index 0921b855..055d430a 100644 --- a/src/tbb/src/tbbmalloc/large_objects.cpp +++ b/src/tbb/src/tbbmalloc/large_objects.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,20 +12,54 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +*/ +#include "tbbmalloc_internal.h" +#include "tbb/tbb_environment.h" +/******************************* Allocation of large objects *********************************************/ +namespace rml { +namespace internal { -*/ +/* ---------------------------- Large Object cache init section ---------------------------------------- */ -#include "tbbmalloc_internal.h" +void LargeObjectCache::init(ExtMemoryPool *memPool) +{ + extMemPool = memPool; + // scalable_allocation_mode can be called before allocator initialization, respect this manual request + if (hugeSizeThreshold == 0) { + // Huge size threshold initialization if environment variable was set + long requestedThreshold = tbb::internal::GetIntegralEnvironmentVariable("TBB_MALLOC_SET_HUGE_SIZE_THRESHOLD"); + // Read valid env or initialize by default with max possible values + if (requestedThreshold != -1) { + setHugeSizeThreshold(requestedThreshold); + } else { + setHugeSizeThreshold(maxHugeSize); + } + } +} -/********* Allocation of large objects ************/ +/* ----------------------------- Huge size threshold settings ----------------------------------------- */ +void LargeObjectCache::setHugeSizeThreshold(size_t value) +{ + // Valid in the huge cache range: [MaxLargeSize, MaxHugeSize]. + if (value <= maxHugeSize) { + hugeSizeThreshold = value >= maxLargeSize ? alignToBin(value) : maxLargeSize; -namespace rml { -namespace internal { + // Calculate local indexes for the global threshold size (for fast search inside a regular cleanup) + largeCache.hugeSizeThresholdIdx = LargeCacheType::numBins; + hugeCache.hugeSizeThresholdIdx = HugeCacheType::sizeToIdx(hugeSizeThreshold); + } +} +bool LargeObjectCache::sizeInCacheRange(size_t size) +{ + return size <= maxHugeSize && (size <= defaultMaxHugeSize || size >= hugeSizeThreshold); +} + +/* ----------------------------------------------------------------------------------------------------- */ /* The functor called by the aggregator for the operation list */ template @@ -101,23 +135,7 @@ class CacheBinFunctor { uintptr_t getCurrTime() const { return currTime; } }; -// ---------------- Cache Bin Aggregator Operation Helpers ---------------- // -// The list of possible operations. -enum CacheBinOperationType { - CBOP_INVALID = 0, - CBOP_GET, - CBOP_PUT_LIST, - CBOP_CLEAN_TO_THRESHOLD, - CBOP_CLEAN_ALL, - CBOP_UPDATE_USED_SIZE -}; - -// The operation status list. CBST_NOWAIT can be specified for non-blocking operations. -enum CacheBinOperationStatus { - CBST_WAIT = 0, - CBST_NOWAIT, - CBST_DONE -}; +/* ---------------- Cache Bin Aggregator Operation Helpers ---------------- */ // The list of structures which describe the operation data struct OpGet { @@ -187,7 +205,8 @@ template OpTypeData& opCast(CacheBinOperation &op) { return *reinterpret_cast(&op.data); } -// ------------------------------------------------------------------------ // + +/* ------------------------------------------------------------------------ */ #if __TBB_MALLOC_LOCACHE_STAT intptr_t mallocCalls, cacheHits; @@ -292,7 +311,7 @@ template void CacheBinFunctor:: num += 1; STAT_increment(getThreadId(), ThreadCommonCounters, cacheLargeObj); - } while (( curr = curr->next )); + } while ((curr = curr->next) != NULL); LargeMemoryBlock *tail = prev; addToPutList(head, tail, num); @@ -386,7 +405,7 @@ template void CacheBinFunctor::operator()(CacheBinOperati #if __TBB_MALLOC_WHITEBOX_TEST tbbmalloc_whitebox::locPutProcessed+=prep.putListNum; #endif - toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum); + toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum, extMemPool->loc.hugeSizeThreshold); } needCleanup = extMemPool->loc.isCleanupNeededOnRange(timeRange, startTime); currTime = endTime - 1; @@ -401,7 +420,7 @@ template void CacheBinFunctor::operator()(CacheBinOperati CacheBinOperation *opNext = opClean->next; prep.commitOperation( opClean ); - while (( opClean = opNext )) { + while ((opClean = opNext) != NULL) { opNext = opClean->next; prep.commitOperation(opClean); } @@ -418,11 +437,13 @@ template void LargeObjectCacheImpl:: CacheBinFunctor func( this, extMemPool, bitMask, idx ); aggregator.execute( op, func, longLifeTime ); - if ( LargeMemoryBlock *toRelease = func.getToRelease() ) + if ( LargeMemoryBlock *toRelease = func.getToRelease()) { extMemPool->backend.returnLargeObject(toRelease); + } - if ( func.isCleanupNeeded() ) + if ( func.isCleanupNeeded() ) { extMemPool->loc.doCleanup( func.getCurrTime(), /*doThreshDecr=*/false); + } } template LargeMemoryBlock *LargeObjectCacheImpl:: @@ -492,22 +513,24 @@ template bool LargeObjectCacheImpl:: } template void LargeObjectCacheImpl:: - CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) { + CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) +{ OpUpdateUsedSize data = {size}; CacheBinOperation op(data); ExecuteOperation( &op, extMemPool, bitMask, idx ); } -/* ----------------------------------------------------------------------------------------------------- */ + /* ------------------------------ Unsafe methods used with the aggregator ------------------------------ */ + template LargeMemoryBlock *LargeObjectCacheImpl:: - CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num) + CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num, size_t hugeSizeThreshold) { size_t size = head->unalignedSize; usedSize -= num*size; MALLOC_ASSERT( !last || (last->age != 0 && last->age != -1U), ASSERT_TEXT ); MALLOC_ASSERT( (tail==head && num==1) || (tail!=head && num>1), ASSERT_TEXT ); LargeMemoryBlock *toRelease = NULL; - if (!lastCleanedAge) { + if (size < hugeSizeThreshold && !lastCleanedAge) { // 1st object of such size was released. // Not cache it, and remember when this occurs // to take into account during cache miss. @@ -560,7 +583,6 @@ template LargeMemoryBlock *LargeObjectCacheImpl:: return result; } -// forget the history for the bin if it was unused for long time template void LargeObjectCacheImpl:: CacheBin::forgetOutdatedState(uintptr_t currTime) { @@ -574,9 +596,9 @@ template void LargeObjectCacheImpl:: bool doCleanup = false; if (ageThreshold) - doCleanup = sinceLastGet > Props::LongWaitFactor*ageThreshold; + doCleanup = sinceLastGet > Props::LongWaitFactor * ageThreshold; else if (lastCleanedAge) - doCleanup = sinceLastGet > Props::LongWaitFactor*(lastCleanedAge - lastGet); + doCleanup = sinceLastGet > Props::LongWaitFactor * (lastCleanedAge - lastGet); if (doCleanup) { lastCleanedAge = 0; @@ -639,6 +661,7 @@ template LargeMemoryBlock *LargeObjectCacheImpl:: return toRelease; } + /* ----------------------------------------------------------------------------------------------------- */ template size_t LargeObjectCacheImpl:: @@ -656,34 +679,40 @@ template size_t LargeObjectCacheImpl:: return cachedSize; } -// release from cache blocks that are older than ageThreshold +// Release objects from cache blocks that are older than ageThreshold template bool LargeObjectCacheImpl::regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currTime, bool doThreshDecr) { bool released = false; BinsSummary binsSummary; - for (int i = bitMask.getMaxTrue(numBins-1); i >= 0; - i = bitMask.getMaxTrue(i-1)) { + // Threshold settings is below this cache or starts from zero index + if (hugeSizeThresholdIdx == 0) return false; + + // Starting searching for bin that is less than huge size threshold (can be cleaned-up) + int startSearchIdx = hugeSizeThresholdIdx - 1; + + for (int i = bitMask.getMaxTrue(startSearchIdx); i >= 0; i = bitMask.getMaxTrue(i-1)) { bin[i].updateBinsSummary(&binsSummary); - if (!doThreshDecr && tooLargeLOC>2 && binsSummary.isLOCTooLarge()) { + if (!doThreshDecr && tooLargeLOC > 2 && binsSummary.isLOCTooLarge()) { // if LOC is too large for quite long time, decrease the threshold // based on bin hit statistics. // For this, redo cleanup from the beginning. // Note: on this iteration total usedSz can be not too large // in comparison to total cachedSz, as we calculated it only // partially. We are ok with it. - i = bitMask.getMaxTrue(numBins-1)+1; + i = bitMask.getMaxTrue(startSearchIdx)+1; doThreshDecr = true; binsSummary.reset(); continue; } if (doThreshDecr) bin[i].decreaseThreshold(); - if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) + + if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) { released = true; + } } - // We want to find if LOC was too large for some time continuously, // so OK with races between incrementing and zeroing, but incrementing // must be atomic. @@ -698,11 +727,20 @@ template bool LargeObjectCacheImpl::cleanAll(ExtMemoryPool *extMemPool) { bool released = false; - for (int i = numBins-1; i >= 0; i--) + for (int i = numBins-1; i >= 0; i--) { released |= bin[i].releaseAllToBackend(extMemPool, &bitMask, i); + } return released; } +template +void LargeObjectCacheImpl::reset() { + tooLargeLOC = 0; + for (int i = numBins-1; i >= 0; i--) + bin[i].init(); + bitMask.reset(); +} + #if __TBB_MALLOC_WHITEBOX_TEST template size_t LargeObjectCacheImpl::getLOCSize() const @@ -764,11 +802,16 @@ bool LargeObjectCache::cleanAll() return largeCache.cleanAll(extMemPool) | hugeCache.cleanAll(extMemPool); } +void LargeObjectCache::reset() +{ + largeCache.reset(); + hugeCache.reset(); +} + template LargeMemoryBlock *LargeObjectCacheImpl::get(ExtMemoryPool *extMemoryPool, size_t size) { - MALLOC_ASSERT( size%Props::CacheStep==0, ASSERT_TEXT ); - int idx = sizeToIdx(size); + int idx = Props::sizeToIdx(size); LargeMemoryBlock *lmb = bin[idx].get(extMemoryPool, size, &bitMask, idx); @@ -782,7 +825,7 @@ LargeMemoryBlock *LargeObjectCacheImpl::get(ExtMemoryPool *extMemoryPool, template void LargeObjectCacheImpl::updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size) { - int idx = sizeToIdx(size); + int idx = Props::sizeToIdx(size); MALLOC_ASSERT(idx void LargeObjectCacheImpl::putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *toCache) { - int toBinIdx = sizeToIdx(toCache->unalignedSize); + int toBinIdx = Props::sizeToIdx(toCache->unalignedSize); MALLOC_ITT_SYNC_RELEASING(bin+toBinIdx); bin[toBinIdx].putList(extMemPool, toCache, &bitMask, toBinIdx); @@ -822,19 +865,33 @@ void LargeObjectCache::updateCacheState(DecreaseOrIncrease op, size_t size) hugeCache.updateCacheState(extMemPool, op, size); } +uintptr_t LargeObjectCache::getCurrTime() +{ + return (uintptr_t)AtomicIncrement((intptr_t&)cacheCurrTime); +} + +uintptr_t LargeObjectCache::getCurrTimeRange(uintptr_t range) +{ + return (uintptr_t)AtomicAdd((intptr_t&)cacheCurrTime, range) + 1; +} + void LargeObjectCache::registerRealloc(size_t oldSize, size_t newSize) { updateCacheState(decrease, oldSize); - updateCacheState(increase, newSize); + updateCacheState(increase, alignToBin(newSize)); +} + +size_t LargeObjectCache::alignToBin(size_t size) { + return size < maxLargeSize ? LargeCacheType::alignToBin(size) : HugeCacheType::alignToBin(size); } -// return artificial bin index, it's used only during sorting and never saved +// Used for internal purpose int LargeObjectCache::sizeToIdx(size_t size) { - MALLOC_ASSERT(size < maxHugeSize, ASSERT_TEXT); - return size < maxLargeSize? + MALLOC_ASSERT(size <= maxHugeSize, ASSERT_TEXT); + return size < maxLargeSize ? LargeCacheType::sizeToIdx(size) : - LargeCacheType::getNumBins()+HugeCacheType::sizeToIdx(size); + LargeCacheType::numBins + HugeCacheType::sizeToIdx(size); } void LargeObjectCache::putList(LargeMemoryBlock *list) @@ -844,7 +901,7 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) for (LargeMemoryBlock *curr = list; curr; curr = toProcess) { LargeMemoryBlock *tail = curr; toProcess = curr->next; - if (curr->unalignedSize >= maxHugeSize) { + if (!sizeInCacheRange(curr->unalignedSize)) { extMemPool->backend.returnLargeObject(curr); continue; } @@ -877,23 +934,23 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) void LargeObjectCache::put(LargeMemoryBlock *largeBlock) { - if (largeBlock->unalignedSize < maxHugeSize) { + size_t blockSize = largeBlock->unalignedSize; + if (sizeInCacheRange(blockSize)) { largeBlock->next = NULL; - if (largeBlock->unalignedSizebackend.returnLargeObject(largeBlock); + } } LargeMemoryBlock *LargeObjectCache::get(size_t size) { - MALLOC_ASSERT( size%largeBlockCacheStep==0, ASSERT_TEXT ); - MALLOC_ASSERT( size>=minLargeSize, ASSERT_TEXT ); - - if ( size < maxHugeSize) { - return size < maxLargeSize? + MALLOC_ASSERT( size >= minLargeSize, ASSERT_TEXT ); + if (sizeInCacheRange(size)) { + return size < maxLargeSize ? largeCache.get(extMemPool, size) : hugeCache.get(extMemPool, size); } return NULL; @@ -963,7 +1020,7 @@ void *ExtMemoryPool::remap(void *ptr, size_t oldSize, size_t newSize, size_t ali void *o = backend.remap(ptr, oldSize, newSize, alignment); if (o) { LargeMemoryBlock *lmb = ((LargeObjectHdr*)o - 1)->memoryBlock; - loc.registerRealloc(lmb->unalignedSize, oldUnalignedSize); + loc.registerRealloc(oldUnalignedSize, lmb->unalignedSize); } return o; } diff --git a/src/tbb/src/tbbmalloc/large_objects.h b/src/tbb/src/tbbmalloc/large_objects.h new file mode 100644 index 00000000..520c9c97 --- /dev/null +++ b/src/tbb/src/tbbmalloc/large_objects.h @@ -0,0 +1,368 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + +#ifndef __TBB_large_objects_H +#define __TBB_large_objects_H + +//! The list of possible Cache Bin Aggregator operations. +/* Declared here to avoid Solaris Studio* 12.2 "multiple definitions" error */ +enum CacheBinOperationType { + CBOP_INVALID = 0, + CBOP_GET, + CBOP_PUT_LIST, + CBOP_CLEAN_TO_THRESHOLD, + CBOP_CLEAN_ALL, + CBOP_UPDATE_USED_SIZE +}; + +// The Cache Bin Aggregator operation status list. +// CBST_NOWAIT can be specified for non-blocking operations. +enum CacheBinOperationStatus { + CBST_WAIT = 0, + CBST_NOWAIT, + CBST_DONE +}; + +/* + * Bins that grow with arithmetic step + */ +template +struct LargeBinStructureProps { +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const size_t CacheStep = 8 * 1024; + static const unsigned NumBins = (MaxSize - MinSize) / CacheStep; + + static size_t alignToBin(size_t size) { + return alignUp(size, CacheStep); + } + + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size < MaxSize, ASSERT_TEXT); + MALLOC_ASSERT(size % CacheStep == 0, ASSERT_TEXT); + return (size - MinSize) / CacheStep; + } +}; + +/* + * Bins that grow with special geometric progression. + */ +template +struct HugeBinStructureProps { + +private: + // Sizes grow with the following formula: Size = MinSize * (2 ^ (Index / StepFactor)) + // There are StepFactor bins (8 be default) between each power of 2 bin + static const int MaxSizeExp = Log2::value; + static const int MinSizeExp = Log2::value; + static const int StepFactor = 8; + static const int StepFactorExp = Log2::value; + +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const unsigned NumBins = (MaxSizeExp - MinSizeExp) * StepFactor; + + static size_t alignToBin(size_t size) { + size_t minorStepExp = BitScanRev(size) - StepFactorExp; + return alignUp(size, 1ULL << minorStepExp); + } + + // Sizes between the power of 2 values are aproximated to StepFactor. + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size <= MaxSize, ASSERT_TEXT); + int sizeExp = (int)BitScanRev(size); // same as __TBB_Log2 + size_t majorStepSize = 1ULL << sizeExp; + int minorStepExp = sizeExp - StepFactorExp; + int minorIdx = (size - majorStepSize) >> minorStepExp; + MALLOC_ASSERT(size == majorStepSize + ((size_t)minorIdx << minorStepExp), + "Size is not aligned on the bin"); + return StepFactor * (sizeExp - MinSizeExp) + minorIdx; + } +}; + +/* + * Cache properties accessor + * + * TooLargeFactor -- when cache size treated "too large" in comparison to user data size + * OnMissFactor -- If cache miss occurred and cache was cleaned, + * set ageThreshold to OnMissFactor * the difference + * between current time and last time cache was cleaned. + * LongWaitFactor -- to detect rarely-used bins and forget about their usage history + */ +template +struct LargeObjectCacheProps : public StructureProps { + static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, LongWaitFactor = LONG_WAIT; +}; + +template +class LargeObjectCacheImpl { +private: + + // Current sizes of used and cached objects. It's calculated while we are + // traversing bins, and used for isLOCTooLarge() check at the same time. + class BinsSummary { + size_t usedSz; + size_t cachedSz; + public: + BinsSummary() : usedSz(0), cachedSz(0) {} + // "too large" criteria + bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor * usedSz; } + void update(size_t usedSize, size_t cachedSize) { + usedSz += usedSize; + cachedSz += cachedSize; + } + void reset() { usedSz = cachedSz = 0; } + }; + +public: + // The number of bins to cache large/huge objects. + static const uint32_t numBins = Props::NumBins; + + typedef BitMaskMax BinBitMask; + + // 2-linked list of same-size cached blocks ordered by age (oldest on top) + // TODO: are we really want the list to be 2-linked? This allows us + // reduce memory consumption and do less operations under lock. + // TODO: try to switch to 32-bit logical time to save space in CacheBin + // and move bins to different cache lines. + class CacheBin { + private: + LargeMemoryBlock *first, + *last; + /* age of an oldest block in the list; equal to last->age, if last defined, + used for quick checking it without acquiring the lock. */ + uintptr_t oldest; + /* currAge when something was excluded out of list because of the age, + not because of cache hit */ + uintptr_t lastCleanedAge; + /* Current threshold value for the blocks of a particular size. + Set on cache miss. */ + intptr_t ageThreshold; + + /* total size of all objects corresponding to the bin and allocated by user */ + size_t usedSize, + /* total size of all objects cached in the bin */ + cachedSize; + /* mean time of presence of block in the bin before successful reuse */ + intptr_t meanHitRange; + /* time of last get called for the bin */ + uintptr_t lastGet; + + typename MallocAggregator::type aggregator; + + void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); + + /* should be placed in zero-initialized memory, ctor not needed. */ + CacheBin(); + + public: + void init() { + memset(this, 0, sizeof(CacheBin)); + } + + /* ---------- Cache accessors ---------- */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + + /* ---------- Cleanup functions -------- */ + bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); + bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); + /* ------------------------------------- */ + + void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + void decreaseThreshold() { + if (ageThreshold) + ageThreshold = (ageThreshold + meanHitRange) / 2; + } + void updateBinsSummary(BinsSummary *binsSummary) const { + binsSummary->update(usedSize, cachedSize); + } + size_t getSize() const { return cachedSize; } + size_t getUsedSize() const { return usedSize; } + size_t reportStat(int num, FILE *f); + + /* --------- Unsafe methods used with the aggregator ------- */ + void forgetOutdatedState(uintptr_t currTime); + LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, + int idx, int num, size_t hugeObjectThreshold); + LargeMemoryBlock *get(); + LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); + LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); + void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { + if (!usedSize) bitMask->set(idx, true); + usedSize += size; + if (!usedSize && !first) bitMask->set(idx, false); + } + void updateMeanHitRange( intptr_t hitRange ) { + hitRange = hitRange >= 0 ? hitRange : 0; + meanHitRange = meanHitRange ? (meanHitRange + hitRange) / 2 : hitRange; + } + void updateAgeThreshold( uintptr_t currTime ) { + if (lastCleanedAge) + ageThreshold = Props::OnMissFactor*(currTime - lastCleanedAge); + } + void updateCachedSize(size_t size) { + cachedSize += size; + } + void setLastGet( uintptr_t newLastGet ) { + lastGet = newLastGet; + } + /* -------------------------------------------------------- */ + }; + + // Huge bins index for fast regular cleanup searching in case of + // the "huge size threshold" setting defined + intptr_t hugeSizeThresholdIdx; + +private: + // How many times LOC was "too large" + intptr_t tooLargeLOC; + // for fast finding of used bins and bins with non-zero usedSize; + // indexed from the end, as we need largest 1st + BinBitMask bitMask; + // bins with lists of recently freed large blocks cached for re-use + CacheBin bin[numBins]; + +public: + /* ------------ CacheBin structure dependent stuff ------------ */ + static size_t alignToBin(size_t size) { + return Props::alignToBin(size); + } + static int sizeToIdx(size_t size) { + return Props::sizeToIdx(size); + } + + /* --------- Main cache functions (put, get object) ------------ */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); + + /* ------------------------ Cleanup ---------------------------- */ + bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); + bool cleanAll(ExtMemoryPool *extMemPool); + + /* -------------------------- Other ---------------------------- */ + void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); + + void reset(); + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif +}; + +class LargeObjectCache { +private: + // Large bins [minLargeSize, maxLargeSize) + // Huge bins [maxLargeSize, maxHugeSize) + static const size_t minLargeSize = 8 * 1024, + maxLargeSize = 8 * 1024 * 1024, + // Cache memory up to 1TB (or 2GB for 32-bit arch), but sieve objects from the special threshold + maxHugeSize = tbb::internal::select_size_t_constant<2147483648U, 1099511627776ULL>::value; + +public: + // Upper bound threshold for caching size. After that size all objects sieve through cache + // By default - 64MB, previous value was 129MB (needed by some Intel(R) Math Kernel Library (Intel(R) MKL) benchmarks) + static const size_t defaultMaxHugeSize = 64UL * 1024UL * 1024UL; + // After that size large object interpreted as huge and does not participate in regular cleanup. + // Can be changed during the program execution. + size_t hugeSizeThreshold; + +private: + // Large objects cache properties + typedef LargeBinStructureProps LargeBSProps; + typedef LargeObjectCacheProps LargeCacheTypeProps; + + // Huge objects cache properties + typedef HugeBinStructureProps HugeBSProps; + typedef LargeObjectCacheProps HugeCacheTypeProps; + + // Cache implementation type with properties + typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; + typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; + + // Beginning of largeCache is more actively used and smaller than hugeCache, + // so put hugeCache first to prevent false sharing + // with LargeObjectCache's predecessor + HugeCacheType hugeCache; + LargeCacheType largeCache; + + /* logical time, incremented on each put/get operation + To prevent starvation between pools, keep separately for each pool. + Overflow is OK, as we only want difference between + its current value and some recent. + + Both malloc and free should increment logical time, as in + a different case multiple cached blocks would have same age, + and accuracy of predictors suffers. + */ + uintptr_t cacheCurrTime; + + // Memory pool that owns this LargeObjectCache. + // strict 1:1 relation, never changed + ExtMemoryPool *extMemPool; + + // Returns artificial bin index, + // it's used only during sorting and never saved + static int sizeToIdx(size_t size); + + // Our friends + friend class Backend; + +public: + void init(ExtMemoryPool *memPool); + + // Item accessors + void put(LargeMemoryBlock *largeBlock); + void putList(LargeMemoryBlock *head); + LargeMemoryBlock *get(size_t size); + + void updateCacheState(DecreaseOrIncrease op, size_t size); + bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); + + // Cleanup operations + bool doCleanup(uintptr_t currTime, bool doThreshDecr); + bool decreasingCleanup(); + bool regularCleanup(); + bool cleanAll(); + void reset(); + + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif + + // Cache deals with exact-fit sizes, so need to align each size + // to the specific bin when put object to cache + static size_t alignToBin(size_t size); + + void setHugeSizeThreshold(size_t value); + + // Check if we should cache or sieve this size + bool sizeInCacheRange(size_t size); + + uintptr_t getCurrTime(); + uintptr_t getCurrTimeRange(uintptr_t range); + void registerRealloc(size_t oldSize, size_t newSize); +}; + +#endif // __TBB_large_objects_H + diff --git a/src/tbb/src/tbbmalloc/lin32-proxy-export.def b/src/tbb/src/tbbmalloc/lin32-proxy-export.def index b3e2849b..99953d54 100644 --- a/src/tbb/src/tbbmalloc/lin32-proxy-export.def +++ b/src/tbb/src/tbbmalloc/lin32-proxy-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def index 33eff0f1..c6dd05c7 100644 --- a/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/lin32-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -49,6 +45,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvj; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvjj; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEjj; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local: diff --git a/src/tbb/src/tbbmalloc/lin64-proxy-export.def b/src/tbb/src/tbbmalloc/lin64-proxy-export.def index 12d8d43d..4f121a84 100644 --- a/src/tbb/src/tbbmalloc/lin64-proxy-export.def +++ b/src/tbb/src/tbbmalloc/lin64-proxy-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def index a57c933b..3ad09cb8 100644 --- a/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/lin64-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -49,6 +45,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local: diff --git a/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def b/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def index 12d8d43d..4f121a84 100644 --- a/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def +++ b/src/tbb/src/tbbmalloc/lin64ipf-proxy-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { diff --git a/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def index a57c933b..64015a2f 100644 --- a/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/lin64ipf-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -37,6 +33,9 @@ __TBB_malloc_safer_aligned_realloc; __TBB_malloc_safer_free; __TBB_malloc_safer_msize; __TBB_malloc_safer_realloc; +/* For tbbmalloc proxy to use MallocMutex with new_handler feature */ +__TBB_machine_lockbyte; +__TBB_machine_trylockbyte; /* memory pool stuff */ _ZN3rml11pool_createElPKNS_13MemPoolPolicyE; @@ -49,6 +48,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local: diff --git a/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def index 71cc1079..0a9d0e3d 100644 --- a/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/mac32-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ _scalable_calloc @@ -46,3 +42,5 @@ __ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm __ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm __ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm __ZN3rml13pool_identifyEPv +__ZN3rml10pool_msizeEPNS_10MemoryPoolEPv + diff --git a/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def index 71cc1079..0a9d0e3d 100644 --- a/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/mac64-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ _scalable_calloc @@ -46,3 +42,5 @@ __ZN3rml12pool_reallocEPNS_10MemoryPoolEPvm __ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvmm __ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEmm __ZN3rml13pool_identifyEPv +__ZN3rml10pool_msizeEPNS_10MemoryPoolEPv + diff --git a/src/tbb/src/tbbmalloc/proxy.cpp b/src/tbb/src/tbbmalloc/proxy.cpp index c865bd1a..d96ae7a0 100644 --- a/src/tbb/src/tbbmalloc/proxy.cpp +++ b/src/tbb/src/tbbmalloc/proxy.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,14 +12,32 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ +#if __linux__ && !__ANDROID__ +// include indirectly so that is not included +#include +// include indirectly so that is not included +#include +// Working around compiler issue with Anaconda's gcc 7.3 compiler package. +// New gcc ported for old libc may provide their inline implementation +// of aligned_alloc as required by new C++ standard, this makes it hard to +// redefine aligned_alloc here. However, running on systems with new libc +// version, it still needs it to be redefined, thus tricking system headers +#if defined(__GLIBC_PREREQ) && !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC +// tell that there is no aligned_alloc +#undef _GLIBCXX_HAVE_ALIGNED_ALLOC +// trick to define another symbol instead +#define aligned_alloc __hidden_redefined_aligned_alloc +// Fix the state and undefine the trick +#include +#undef aligned_alloc +#endif // defined(__GLIBC_PREREQ)&&!__GLIBC_PREREQ(2, 16)&&_GLIBCXX_HAVE_ALIGNED_ALLOC +#endif // __linux__ && !__ANDROID__ + #include "proxy.h" #include "tbb/tbb_config.h" +#include "tbb/tbb_environment.h" #if !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) && !defined(__SUNPRO_CC) #if TBB_USE_EXCEPTIONS @@ -31,6 +49,78 @@ #define TBB_USE_EXCEPTIONS 1 #endif +#if __TBB_CPP11_PRESENT +#define __TBB_THROW_BAD_ALLOC +#define __TBB_NO_THROW noexcept +#else +#define __TBB_THROW_BAD_ALLOC throw(std::bad_alloc) +#define __TBB_NO_THROW throw() +#endif + +#if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT +/*** internal global operator new implementation (Linux, Windows) ***/ +#include + +// Synchronization primitives to protect original library pointers and new_handler +#include "Synchronize.h" + +#if __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT +// Use MallocMutex implementation +typedef MallocMutex ProxyMutex; +#else +// One byte atomic intrinsics are not available, +// so use simple pointer based spin mutex +class SimpleSpinMutex : tbb::internal::no_copy { + intptr_t flag; +public: + class scoped_lock : tbb::internal::no_copy { + SimpleSpinMutex& mutex; + public: + scoped_lock( SimpleSpinMutex& m ) : mutex(m) { + while( !(AtomicFetchStore( &(m.flag), 1 ) == 0) ); + } + ~scoped_lock() { + FencedStore(mutex.flag, 0); + } + }; + friend class scoped_lock; +}; +typedef SimpleSpinMutex ProxyMutex; +#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ + +// In case there is no std::get_new_handler function +// which provides synchronized access to std::new_handler +#if !__TBB_CPP11_GET_NEW_HANDLER_PRESENT +static ProxyMutex new_lock; +#endif + +static inline void* InternalOperatorNew(size_t sz) { + void* res = scalable_malloc(sz); +#if TBB_USE_EXCEPTIONS + while (!res) { + std::new_handler handler; +#if __TBB_CPP11_GET_NEW_HANDLER_PRESENT + handler = std::get_new_handler(); +#else + { + ProxyMutex::scoped_lock lock(new_lock); + handler = std::set_new_handler(0); + std::set_new_handler(handler); + } +#endif + if (handler) { + (*handler)(); + } else { + throw std::bad_alloc(); + } + res = scalable_malloc(sz); +} +#endif /* TBB_USE_EXCEPTIONS */ + return res; +} +/*** end of internal global operator new implementation ***/ +#endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT + #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED #ifndef __THROW @@ -38,7 +128,6 @@ #endif /*** service functions and variables ***/ - #include // for memset #include // for sysconf @@ -50,7 +139,6 @@ static inline void initPageSize() } #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED -#include "Customize.h" // FencedStore #include #include // mallinfo @@ -86,7 +174,7 @@ static intptr_t origFuncSearched; inline void InitOrigPointers() { // race is OK here, as different threads found same functions - if (!origFuncSearched) { + if (!FencedLoad(origFuncSearched)) { orig_free = dlsym(RTLD_NEXT, "free"); orig_realloc = dlsym(RTLD_NEXT, "realloc"); orig_msize = dlsym(RTLD_NEXT, "malloc_usable_size"); @@ -229,43 +317,31 @@ void *__libc_realloc(void *ptr, size_t size) /*** replacements for global operators new and delete ***/ -#include - -void * operator new(size_t sz) { - void *res = scalable_malloc(sz); -#if TBB_USE_EXCEPTIONS - if (NULL == res) - throw std::bad_alloc(); -#endif /* TBB_USE_EXCEPTIONS */ - return res; +void* operator new(size_t sz) __TBB_THROW_BAD_ALLOC { + return InternalOperatorNew(sz); } -void* operator new[](size_t sz) { - void *res = scalable_malloc(sz); -#if TBB_USE_EXCEPTIONS - if (NULL == res) - throw std::bad_alloc(); -#endif /* TBB_USE_EXCEPTIONS */ - return res; +void* operator new[](size_t sz) __TBB_THROW_BAD_ALLOC { + return InternalOperatorNew(sz); } -void operator delete(void* ptr) throw() { +void operator delete(void* ptr) __TBB_NO_THROW { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void operator delete[](void* ptr) throw() { +void operator delete[](void* ptr) __TBB_NO_THROW { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void* operator new(size_t sz, const std::nothrow_t&) throw() { +void* operator new(size_t sz, const std::nothrow_t&) __TBB_NO_THROW { return scalable_malloc(sz); } -void* operator new[](std::size_t sz, const std::nothrow_t&) throw() { +void* operator new[](std::size_t sz, const std::nothrow_t&) __TBB_NO_THROW { return scalable_malloc(sz); } -void operator delete(void* ptr, const std::nothrow_t&) throw() { +void operator delete(void* ptr, const std::nothrow_t&) __TBB_NO_THROW { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } -void operator delete[](void* ptr, const std::nothrow_t&) throw() { +void operator delete[](void* ptr, const std::nothrow_t&) __TBB_NO_THROW { InitOrigPointers(); __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free); } @@ -273,7 +349,6 @@ void operator delete[](void* ptr, const std::nothrow_t&) throw() { #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED */ #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED */ - #ifdef _WIN32 #include @@ -331,19 +406,35 @@ void* __TBB_malloc_safer_realloc_##CRTLIB( void *ptr, size_t size ) return __TBB_malloc_safer_realloc( ptr, size, &func_ptrs ); \ } \ \ -void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t aligment ) \ +void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t alignment ) \ { \ orig_aligned_ptrs func_ptrs = {orig__aligned_free_##CRTLIB, orig__aligned_msize_##CRTLIB}; \ - return __TBB_malloc_safer_aligned_realloc( ptr, size, aligment, &func_ptrs ); \ + return __TBB_malloc_safer_aligned_realloc( ptr, size, alignment, &func_ptrs ); \ } -// Limit is 30 bytes/60 symbols per line, * can be used to match any digit in bytecodes. -// Purpose of the pattern is to mark an instruction bound, it should consist of several -// full instructions plus one more byte. It's not required for the patterns to be unique -// (i.e., it's OK to have same pattern for unrelated functions). +// Only for ucrtbase: substitution for _o_free +void (*orig__o_free)(void*); +void __TBB_malloc__o_free(void *ptr) +{ + __TBB_malloc_safer_free( ptr, orig__o_free ); +} +// Only for ucrtbase: substitution for _free_base +void(*orig__free_base)(void*); +void __TBB_malloc__free_base(void *ptr) +{ + __TBB_malloc_safer_free(ptr, orig__free_base); +} + +// Size limit is MAX_PATTERN_SIZE (28) byte codes / 56 symbols per line. +// * can be used to match any digit in byte codes. +// # followed by several * indicate a relative address that needs to be corrected. +// Purpose of the pattern is to mark an instruction bound; it should consist of several +// full instructions plus one extra byte code. It's not required for the patterns +// to be unique (i.e., it's OK to have same pattern for unrelated functions). // TODO: use hot patch prologues if exist const char* known_bytecodes[] = { #if _WIN64 +// "========================================================" - 56 symbols "4883EC284885C974", // release free() "4883EC284885C975", // release _msize() "4885C974375348", // release free() 8.0.50727.42, 10.0 @@ -351,15 +442,18 @@ const char* known_bytecodes[] = { "C7442410000000008B", // release free() ucrtbase.dll 10.0.14393.33 "E90B000000CCCC", // release _msize() ucrtbase.dll 10.0.14393.33 "48895C24085748", // release _aligned_msize() ucrtbase.dll 10.0.14393.33 + "E903000000CCCC", // release _aligned_msize() ucrtbase.dll 10.0.16299.522 "48894C24084883EC28BA", // debug prologue "4C894424184889542410", // debug _aligned_msize() 10.0 "48894C24084883EC2848", // debug _aligned_free 10.0 + "488BD1488D0D#*******E9", // _o_free(), ucrtbase.dll #if __TBB_OVERLOAD_OLD_MSVCR "48895C2408574883EC3049", // release _aligned_msize 9.0 "4883EC384885C975", // release _msize() 9.0 "4C8BC1488B0DA6E4040033", // an old win64 SDK #endif #else // _WIN32 +// "========================================================" - 56 symbols "8BFF558BEC8B", // multiple "8BFF558BEC83", // release free() & _msize() 10.0.40219.325, _msize() ucrtbase.dll "8BFF558BECFF", // release _aligned_msize ucrtbase.dll @@ -429,46 +523,41 @@ __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120d); __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120); __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(ucrtbase); - /*** replacements for global operators new and delete ***/ -#include - #if _MSC_VER && !defined(__INTEL_COMPILER) #pragma warning( push ) #pragma warning( disable : 4290 ) #endif -void * operator_new(size_t sz) { - void *res = scalable_malloc(sz); - if (NULL == res) throw std::bad_alloc(); - return res; +/*** operator new overloads internals (Linux, Windows) ***/ + +void* operator_new(size_t sz) __TBB_THROW_BAD_ALLOC { + return InternalOperatorNew(sz); } -void* operator_new_arr(size_t sz) { - void *res = scalable_malloc(sz); - if (NULL == res) throw std::bad_alloc(); - return res; +void* operator_new_arr(size_t sz) __TBB_THROW_BAD_ALLOC { + return InternalOperatorNew(sz); } -void operator_delete(void* ptr) throw() { +void operator_delete(void* ptr) __TBB_NO_THROW { __TBB_malloc_safer_delete(ptr); } #if _MSC_VER && !defined(__INTEL_COMPILER) #pragma warning( pop ) #endif -void operator_delete_arr(void* ptr) throw() { +void operator_delete_arr(void* ptr) __TBB_NO_THROW { __TBB_malloc_safer_delete(ptr); } -void* operator_new_t(size_t sz, const std::nothrow_t&) throw() { +void* operator_new_t(size_t sz, const std::nothrow_t&) __TBB_NO_THROW { return scalable_malloc(sz); } -void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) throw() { +void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) __TBB_NO_THROW { return scalable_malloc(sz); } -void operator_delete_t(void* ptr, const std::nothrow_t&) throw() { +void operator_delete_t(void* ptr, const std::nothrow_t&) __TBB_NO_THROW { __TBB_malloc_safer_delete(ptr); } -void operator_delete_arr_t(void* ptr, const std::nothrow_t&) throw() { +void operator_delete_arr_t(void* ptr, const std::nothrow_t&) __TBB_NO_THROW { __TBB_malloc_safer_delete(ptr); } @@ -577,7 +666,7 @@ bool BytecodesAreKnown(const unicode_char_t *dllName) if (!module) return false; for (int i=0; funcName[i]; i++) - if (! IsPrologueKnown(module, funcName[i], known_bytecodes)) { + if (! IsPrologueKnown(dllName, funcName[i], known_bytecodes, module)) { fprintf(stderr, "TBBmalloc: skip allocation functions replacement in " WCHAR_SPEC ": unknown prologue for function " WCHAR_SPEC "\n", dllName, funcName[i]); return false; @@ -597,7 +686,7 @@ void SkipReplacement(const unicode_char_t *dllName) char *dllStr = buffer; errno_t ret = wcstombs_s(&real_sz, dllStr, sz, dllName, sz-1); - __TBB_ASSERT(!ret, "Dll name conversion failed") + __TBB_ASSERT(!ret, "Dll name conversion failed"); #endif for (size_t i=0; i +struct Log2 { static const int value = 1 + Log2<(NUM >> 1)>::value; }; +template <> +struct Log2<1> { static const int value = 0; }; + #if defined(min) #undef min #endif @@ -69,6 +73,61 @@ T min ( const T& val1, const T& val2 ) { return val1 < val2 ? val1 : val2; } +/* + * Functions to parse files information (system files for example) + */ + +#include + +#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER) + // Suppress overzealous compiler warnings that default ctor and assignment + // operator cannot be generated and object 'class' can never be instantiated. + #pragma warning(push) + #pragma warning(disable:4510 4512 4610) +#endif + +#if __SUNPRO_CC + // Suppress overzealous compiler warnings that a class with a reference member + // lacks a user-defined constructor, which can lead to errors + #pragma error_messages (off, refmemnoconstr) +#endif + +// TODO: add a constructor to remove warnings suppression +struct parseFileItem { + const char* format; + unsigned long long& value; +}; + +#if defined(_MSC_VER) && (_MSC_VER<1900) && !defined(__INTEL_COMPILER) + #pragma warning(pop) +#endif + +#if __SUNPRO_CC + #pragma error_messages (on, refmemnoconstr) +#endif + +template +void parseFile(const char* file, const parseFileItem (&items)[N]) { + // Tries to find all items in each line + int found[N] = { 0 }; + // If all items found, stop forward file reading + int numFound = 0; + // Line storage + char buf[BUF_LINE_SIZE]; + + if (FILE *f = fopen(file, "r")) { + while (numFound < N && fgets(buf, BUF_LINE_SIZE, f)) { + for (int i = 0; i < N; ++i) { + if (!found[i] && 1 == sscanf(buf, items[i].format, &items[i].value)) { + ++numFound; + found[i] = 1; + } + } + } + fclose(f); + } +} + namespace rml { namespace internal { @@ -85,5 +144,8 @@ const uint32_t estimatedCacheLineSize = 128; const uint32_t estimatedCacheLineSize = 64; #endif -}} // namespaces +} // namespace internal +} // namespace rml + #endif /* __TBB_shared_utils_H */ + diff --git a/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp b/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp index b0157856..7410fce1 100644 --- a/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp +++ b/src/tbb/src/tbbmalloc/tbb_function_replacement.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,28 +12,84 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" #if !__TBB_WIN8UI_SUPPORT && defined(_WIN32) +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE 1 +#endif #define __TBB_NO_IMPLICIT_LINKAGE 1 +// no standard-conforming implementation of snprintf prior to VS 2015 +#if !defined(_MSC_VER) || _MSC_VER>=1900 +#define LOG_PRINT(s, n, format, ...) snprintf(s, n, format, __VA_ARGS__) +#else +#define LOG_PRINT(s, n, format, ...) _snprintf_s(s, n, _TRUNCATE, format, __VA_ARGS__) +#endif + #include #include #include +#include #include "tbb_function_replacement.h" -#include "tbb/tbb_config.h" #include "tbb/tbb_stddef.h" #include "../tbb/tbb_assert_impl.h" +// The information about a standard memory allocation function for the replacement log +struct FunctionInfo { + const char* funcName; + const char* dllName; +}; + +// Namespace that processes and manages the output of records to the Log journal +// that will be provided to user by TBB_malloc_replacement_log() +namespace Log { + // Value of RECORDS_COUNT is set due to the fact that we maximally + // scan 8 modules, and in every module we can swap 6 opcodes. (rounded to 8) + static const unsigned RECORDS_COUNT = 8 * 8; + static const unsigned RECORD_LENGTH = MAX_PATH; + + // Need to add 1 to count of records, because last record must be always NULL + static char *records[RECORDS_COUNT + 1]; + static bool replacement_status = true; + + // Internal counter that contains number of next string for record + static unsigned record_number = 0; + + // Function that writes info about (not)found opcodes to the Log journal + // functionInfo - information about a standard memory allocation function for the replacement log + // opcodeString - string, that contain byte code of this function + // status - information about function replacement status + static void record(FunctionInfo functionInfo, const char * opcodeString, bool status) { + __TBB_ASSERT(functionInfo.dllName, "Empty DLL name value"); + __TBB_ASSERT(functionInfo.funcName, "Empty function name value"); + __TBB_ASSERT(opcodeString, "Empty opcode"); + __TBB_ASSERT(record_number <= RECORDS_COUNT, "Incorrect record number"); + + //If some replacement failed -> set status to false + replacement_status &= status; + + // If we reach the end of the log, write this message to the last line + if (record_number == RECORDS_COUNT) { + // %s - workaround to fix empty variable argument parsing behavior in GCC + LOG_PRINT(records[RECORDS_COUNT - 1], RECORD_LENGTH, "%s", "Log was truncated."); + return; + } + + char* entry = (char*)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, RECORD_LENGTH); + __TBB_ASSERT(entry, "Invalid memory was returned"); + + LOG_PRINT(entry, RECORD_LENGTH, "%s: %s (%s), byte pattern: <%s>", + status ? "Success" : "Fail", functionInfo.funcName, functionInfo.dllName, opcodeString); + + records[record_number++] = entry; + } +}; + inline UINT_PTR Ptr2Addrint(LPVOID ptr) { Int2Ptr i2p; @@ -66,7 +122,7 @@ inline bool IsInDistance(UINT_PTR addr1, UINT_PTR addr2, __int64 dist) * doesn't allocate memory dynamically. * * The struct MemoryBuffer holds the data about a page in the memory used for - * replacing functions in Intel64 where the target is too far to be replaced + * replacing functions in 64-bit code where the target is too far to be replaced * with a short jump. All the calculations of m_base and m_next are in a multiple * of SIZE_OF_ADDRESS (which is 8 in Win64). */ @@ -180,14 +236,13 @@ static MemoryProvider memProvider; // Compare opcodes from dictionary (str1) and opcodes from code (str2) // str1 might contain '*' to mask addresses -// RETURN: NULL if opcodes did not match, string length of str1 on success +// RETURN: 0 if opcodes did not match, 1 on success size_t compareStrings( const char *str1, const char *str2 ) { - size_t str1Length = strlen(str1); - for (size_t i=0; i= SIZE_OF_RELJUMP, "Incorrect bytecode pattern?" ); - UINT_PTR strdAddr = memProvider.GetLocation(srcAddr); - if (!strdAddr) - return 0; - *storedAddr = Addrint2Ptr(strdAddr); - // Set 'executable' flag for original instructions in the new place - DWORD pageFlags = PAGE_EXECUTE_READWRITE; - if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; - // Copy original instructions to the new place - memcpy(*storedAddr, codePtr, opcodesNumber); - // Set jump to the code after replacement - offset = srcAddr - strdAddr - SIZE_OF_RELJUMP; - offset32 = (UINT)((offset & 0xFFFFFFFF)); - *((UCHAR*)*storedAddr+opcodesNumber) = 0xE9; - memcpy(((UCHAR*)*storedAddr+opcodesNumber+1), &offset32, sizeof(offset32)); - } + bytesToMove = strlen(pattern)/2-1; // The last byte matching the pattern must not be copied + __TBB_ASSERT_RELEASE( bytesToMove >= SIZE_OF_RELJUMP, "Incorrect bytecode pattern?" ); + UINT_PTR trampAddr = memProvider.GetLocation(srcAddr); + if (!trampAddr) + return 0; + *storedAddr = Addrint2Ptr(trampAddr); + // Set 'executable' flag for original instructions in the new place + DWORD pageFlags = PAGE_EXECUTE_READWRITE; + if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; + // Copy original instructions to the new place + memcpy(*storedAddr, codePtr, bytesToMove); + offset = srcAddr - trampAddr; + offset32 = (UINT)(offset & 0xFFFFFFFF); + CorrectOffset( trampAddr, pattern, offset32 ); + // Set jump to the code after replacement + offset32 -= SIZE_OF_RELJUMP; + *(UCHAR*)(trampAddr+bytesToMove) = 0xE9; + memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32)); } // The following will work correctly even if srcAddr>tgtAddr, as long as @@ -287,7 +359,7 @@ static DWORD InsertTrampoline32(void *inpAddr, void *targetAddr, const char ** o memcpy(codePtr+1, &offset32, sizeof(offset32)); // Fill the rest with NOPs to correctly see disassembler of old code in debugger. - for( unsigned i=SIZE_OF_RELJUMP; i= SIZE_OF_INDJUMP, "Incorrect bytecode pattern?" ); - UINT_PTR strdAddr = memProvider.GetLocation(srcAddr); - if (!strdAddr) - return 0; - *storedAddr = Addrint2Ptr(strdAddr); - // Set 'executable' flag for original instructions in the new place - DWORD pageFlags = PAGE_EXECUTE_READWRITE; - if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; - // Copy original instructions to the new place - memcpy(*storedAddr, codePtr, opcodesNumber); - // Set jump to the code after replacement. It is within the distance of relative jump! - offset = srcAddr - strdAddr - SIZE_OF_RELJUMP; - offset32 = (UINT)((offset & 0xFFFFFFFF)); - *((UCHAR*)*storedAddr+opcodesNumber) = 0xE9; - memcpy(((UCHAR*)*storedAddr+opcodesNumber+1), &offset32, sizeof(offset32)); - } + bytesToMove = strlen(pattern)/2-1; // The last byte matching the pattern must not be copied + __TBB_ASSERT_RELEASE( bytesToMove >= SIZE_OF_INDJUMP, "Incorrect bytecode pattern?" ); + UINT_PTR trampAddr = memProvider.GetLocation(srcAddr); + if (!trampAddr) + return 0; + *storedAddr = Addrint2Ptr(trampAddr); + // Set 'executable' flag for original instructions in the new place + DWORD pageFlags = PAGE_EXECUTE_READWRITE; + if (!VirtualProtect(*storedAddr, MAX_PROBE_SIZE, pageFlags, &pageFlags)) return 0; + // Copy original instructions to the new place + memcpy(*storedAddr, codePtr, bytesToMove); + offset = srcAddr - trampAddr; + offset32 = (UINT)(offset & 0xFFFFFFFF); + CorrectOffset( trampAddr, pattern, offset32 ); + // Set jump to the code after replacement. It is within the distance of relative jump! + offset32 -= SIZE_OF_RELJUMP; + *(UCHAR*)(trampAddr+bytesToMove) = 0xE9; + memcpy((UCHAR*)(trampAddr+bytesToMove+1), &offset32, sizeof(offset32)); } // Fill the buffer @@ -353,7 +420,7 @@ static DWORD InsertTrampoline64(void *inpAddr, void *targetAddr, const char ** o memcpy(codePtr+2, &offset32, sizeof(offset32)); // Fill the rest with NOPs to correctly see disassembler of old code in debugger. - for( unsigned i=SIZE_OF_INDJUMP; i 0, "abortOnError ignored in CheckOpcodes?" ); + pattern = opcodes[opcodeIdx-1]; // -1 compensates for +1 in CheckOpcodes + } + } + + probeSize = InsertTrampoline32(inpAddr, targetAddr, pattern, origFunc); if (!probeSize) - probeSize = InsertTrampoline64(inpAddr, targetAddr, opcodes, origFunc); + probeSize = InsertTrampoline64(inpAddr, targetAddr, pattern, origFunc); // Restore original protection VirtualProtect(inpAddr, MAX_PROBE_SIZE, origProt, &origProt); @@ -471,12 +556,28 @@ FRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR return FRR_OK; } -bool IsPrologueKnown(HMODULE module, const char *funcName, const char **opcodes) +bool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module) { FARPROC inpFunc = GetProcAddress(module, funcName); - if (!inpFunc) + FunctionInfo functionInfo = { funcName, dllName }; + + if (!inpFunc) { + Log::record(functionInfo, "unknown", /*status*/ false); return false; - return CheckOpcodes( opcodes, (void*)inpFunc, /*abortOnError=*/false ) != 0; + } + + return CheckOpcodes( opcodes, (void*)inpFunc, /*abortOnError=*/false, &functionInfo) != 0; +} + +// Public Windows API +extern "C" __declspec(dllexport) int TBB_malloc_replacement_log(char *** function_replacement_log_ptr) +{ + if (function_replacement_log_ptr != NULL) { + *function_replacement_log_ptr = Log::records; + } + + // If we have no logs -> return false status + return Log::replacement_status && Log::records[0] != NULL ? 0 : -1; } #endif /* !__TBB_WIN8UI_SUPPORT && defined(_WIN32) */ diff --git a/src/tbb/src/tbbmalloc/tbb_function_replacement.h b/src/tbb/src/tbbmalloc/tbb_function_replacement.h index e986ab18..3549db62 100644 --- a/src/tbb/src/tbbmalloc/tbb_function_replacement.h +++ b/src/tbb/src/tbbmalloc/tbb_function_replacement.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_function_replacement_H @@ -45,7 +41,7 @@ typedef void (*FUNCPTR)(); FRR_TYPE ReplaceFunctionA(const char *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=NULL); FRR_TYPE ReplaceFunctionW(const wchar_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc=NULL); -bool IsPrologueKnown(HMODULE module, const char *funcName, const char **opcodes); +bool IsPrologueKnown(const char* dllName, const char *funcName, const char **opcodes, HMODULE module); // Utilities to convert between ADDRESS and LPVOID union Int2Ptr { @@ -56,7 +52,7 @@ union Int2Ptr { inline UINT_PTR Ptr2Addrint(LPVOID ptr); inline LPVOID Addrint2Ptr(UINT_PTR ptr); -// Use this value as the maximum size the trampoline region +// The size of a trampoline region const unsigned MAX_PROBE_SIZE = 32; // The size of a jump relative instruction "e9 00 00 00 00" @@ -68,6 +64,10 @@ const unsigned SIZE_OF_INDJUMP = 6; // The size of address we put in the location (in Intel64) const unsigned SIZE_OF_ADDRESS = 8; +// The size limit (in bytes) for an opcode pattern to fit into a trampoline +// There should be enough space left for a relative jump; +1 is for the extra pattern byte. +const unsigned MAX_PATTERN_SIZE = MAX_PROBE_SIZE - SIZE_OF_RELJUMP + 1; + // The max distance covered in 32 bits: 2^31 - 1 - C // where C should not be smaller than the size of a probe. // The latter is important to correctly handle "backward" jumps. diff --git a/src/tbb/src/tbbmalloc/tbbmalloc.cpp b/src/tbb/src/tbbmalloc/tbbmalloc.cpp index fa0e6984..7de858c7 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc.cpp +++ b/src/tbb/src/tbbmalloc/tbbmalloc.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "TypeDefinitions.h" // Customize.h and proxy.h get included @@ -45,7 +41,7 @@ namespace internal { #define MALLOCLIB_NAME "tbbmalloc" DEBUG_SUFFIX ".dll" #elif __APPLE__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".dylib" -#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX || __ANDROID__ +#elif __FreeBSD__ || __NetBSD__ || __OpenBSD__ || __sun || _AIX || __ANDROID__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX ".so" #elif __linux__ #define MALLOCLIB_NAME "libtbbmalloc" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) @@ -75,16 +71,15 @@ void init_tbbmalloc() { #if !__TBB_SOURCE_DIRECTLY_INCLUDED #if USE_WINTHREAD -extern "C" BOOL WINAPI DllMain( HINSTANCE /*hInst*/, DWORD callReason, LPVOID ) +extern "C" BOOL WINAPI DllMain( HINSTANCE /*hInst*/, DWORD callReason, LPVOID lpvReserved) { - if (callReason==DLL_THREAD_DETACH) { __TBB_mallocThreadShutdownNotification(); } else if (callReason==DLL_PROCESS_DETACH) { - __TBB_mallocProcessShutdownNotification(); + __TBB_mallocProcessShutdownNotification(lpvReserved != NULL); } return TRUE; } @@ -98,7 +93,7 @@ struct RegisterProcessShutdownNotification { } #endif /* !__TBB_USE_DLOPEN_REENTRANCY_WORKAROUND */ ~RegisterProcessShutdownNotification() { - __TBB_mallocProcessShutdownNotification(); + __TBB_mallocProcessShutdownNotification(false); } }; diff --git a/src/tbb/src/tbbmalloc/tbbmalloc.rc b/src/tbb/src/tbbmalloc/tbbmalloc.rc index 2b65e554..6a8b1698 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc.rc +++ b/src/tbb/src/tbbmalloc/tbbmalloc.rc @@ -1,4 +1,4 @@ -// Copyright (c) 2005-2017 Intel Corporation +// Copyright (c) 2005-2019 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// -// -// -// // Microsoft Visual C++ generated resource script. // @@ -84,7 +80,7 @@ BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Scalable Allocator library\0" VALUE "FileVersion", TBBMALLOC_VERSION "\0" - VALUE "LegalCopyright", "Copyright 2005-2017 Intel Corporation. All Rights Reserved.\0" + VALUE "LegalCopyright", "Copyright 2005-2019 Intel Corporation. All Rights Reserved.\0" VALUE "LegalTrademarks", "\0" #ifndef TBB_USE_DEBUG VALUE "OriginalFilename", "tbbmalloc.dll\0" diff --git a/src/tbb/src/tbbmalloc/tbbmalloc_internal.h b/src/tbb/src/tbbmalloc/tbbmalloc_internal.h index e4082bc7..cec0b0bd 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc_internal.h +++ b/src/tbb/src/tbbmalloc/tbbmalloc_internal.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,14 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbbmalloc_internal_H -#define __TBB_tbbmalloc_internal_H 1 +#define __TBB_tbbmalloc_internal_H #include "TypeDefinitions.h" /* Also includes customization layer Customize.h */ @@ -99,10 +95,19 @@ extern intptr_t memAllocKB, memHitKB; template void suppress_unused_warning( const T& ) {} +/********** Various global default constants ********/ + +/* + * Default huge page size + */ +static const size_t HUGE_PAGE_SIZE = 2 * 1024 * 1024; + +/********** End of global default constatns *********/ + /********** Various numeric parameters controlling allocations ********/ /* - * smabSize - the size of a block for allocation of small objects, + * slabSize - the size of a block for allocation of small objects, * it must be larger than maxSegregatedObjectSize. */ const uintptr_t slabSize = 16*1024; @@ -241,7 +246,7 @@ class AllLocalCaches { public: void registerThread(TLSRemote *tls); void unregisterThread(TLSRemote *tls); - bool cleanup(ExtMemoryPool *extPool, bool cleanOnlyUnused); + bool cleanup(bool cleanOnlyUnused); void markUnused(); void reset() { head = NULL; } }; @@ -275,220 +280,8 @@ class OrphanedBlocks { bool cleanup(Backend* backend); }; -/* cache blocks in range [MinSize; MaxSize) in bins with CacheStep - TooLargeFactor -- when cache size treated "too large" in comparison to user data size - OnMissFactor -- If cache miss occurred and cache was cleaned, - set ageThreshold to OnMissFactor * the difference - between current time and last time cache was cleaned. - LongWaitFactor -- to detect rarely-used bins and forget about their usage history -*/ -template -struct LargeObjectCacheProps { - static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; - static const uint32_t CacheStep = CACHE_STEP; - static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, - LongWaitFactor = LONG_WAIT; -}; - -template -class LargeObjectCacheImpl { -private: - // The number of bins to cache large objects. - static const uint32_t numBins = (Props::MaxSize-Props::MinSize)/Props::CacheStep; - // Current sizes of used and cached objects. It's calculated while we are - // traversing bins, and used for isLOCTooLarge() check at the same time. - class BinsSummary { - size_t usedSz; - size_t cachedSz; - public: - BinsSummary() : usedSz(0), cachedSz(0) {} - // "too large" criteria - bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor*usedSz; } - void update(size_t usedSize, size_t cachedSize) { - usedSz += usedSize; - cachedSz += cachedSize; - } - void reset() { usedSz = cachedSz = 0; } - }; -public: - typedef BitMaskMax BinBitMask; - - // 2-linked list of same-size cached blocks ordered by age (oldest on top) - // TODO: are we really want the list to be 2-linked? This allows us - // reduce memory consumption and do less operations under lock. - // TODO: try to switch to 32-bit logical time to save space in CacheBin - // and move bins to different cache lines. - class CacheBin { - private: - LargeMemoryBlock *first, - *last; - /* age of an oldest block in the list; equal to last->age, if last defined, - used for quick cheching it without acquiring the lock. */ - uintptr_t oldest; - /* currAge when something was excluded out of list because of the age, - not because of cache hit */ - uintptr_t lastCleanedAge; - /* Current threshold value for the blocks of a particular size. - Set on cache miss. */ - intptr_t ageThreshold; - - /* total size of all objects corresponding to the bin and allocated by user */ - size_t usedSize, - /* total size of all objects cached in the bin */ - cachedSize; - /* mean time of presence of block in the bin before successful reuse */ - intptr_t meanHitRange; - /* time of last get called for the bin */ - uintptr_t lastGet; - - typename MallocAggregator::type aggregator; - - void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); - /* should be placed in zero-initialized memory, ctor not needed. */ - CacheBin(); - public: - void init() { memset(this, 0, sizeof(CacheBin)); } - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); - bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); - void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - - void decreaseThreshold() { - if (ageThreshold) - ageThreshold = (ageThreshold + meanHitRange)/2; - } - void updateBinsSummary(BinsSummary *binsSummary) const { - binsSummary->update(usedSize, cachedSize); - } - size_t getSize() const { return cachedSize; } - size_t getUsedSize() const { return usedSize; } - size_t reportStat(int num, FILE *f); - /* ---------- unsafe methods used with the aggregator ---------- */ - void forgetOutdatedState(uintptr_t currTime); - LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num); - LargeMemoryBlock *get(); - LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); - LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); - void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { - if (!usedSize) bitMask->set(idx, true); - usedSize += size; - if (!usedSize && !first) bitMask->set(idx, false); - } - void updateMeanHitRange( intptr_t hitRange ) { - hitRange = hitRange >= 0 ? hitRange : 0; - meanHitRange = meanHitRange ? (meanHitRange + hitRange)/2 : hitRange; - } - void updateAgeThreshold( uintptr_t currTime ) { - if (lastCleanedAge) - ageThreshold = Props::OnMissFactor*(currTime - lastCleanedAge); - } - void updateCachedSize(size_t size) { cachedSize += size; } - void setLastGet( uintptr_t newLastGet ) { lastGet = newLastGet; } - /* -------------------------------------------------------- */ - }; -private: - intptr_t tooLargeLOC; // how many times LOC was "too large" - // for fast finding of used bins and bins with non-zero usedSize; - // indexed from the end, as we need largest 1st - BinBitMask bitMask; - // bins with lists of recently freed large blocks cached for re-use - CacheBin bin[numBins]; - -public: - static int sizeToIdx(size_t size) { - MALLOC_ASSERT(Props::MinSize <= size && size < Props::MaxSize, ASSERT_TEXT); - return (size-Props::MinSize)/Props::CacheStep; - } - static int getNumBins() { return numBins; } - - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); - - void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); - bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); - bool cleanAll(ExtMemoryPool *extMemPool); - void reset() { - tooLargeLOC = 0; - for (int i = numBins-1; i >= 0; i--) - bin[i].init(); - bitMask.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif -}; - -class LargeObjectCache { - static const size_t minLargeSize = 8*1024, - maxLargeSize = 8*1024*1024, - // There are benchmarks of interest that should work well with objects of this size - maxHugeSize = 129*1024*1024; -public: - // Difference between object sizes in large block bins - static const uint32_t largeBlockCacheStep = 8*1024, - hugeBlockCacheStep = 512*1024; -private: - typedef LargeObjectCacheProps LargeCacheTypeProps; - typedef LargeObjectCacheProps HugeCacheTypeProps; - typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; - typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; - - // beginning of largeCache is more actively used and smaller than hugeCache, - // so put hugeCache first to prevent false sharing - // with LargeObjectCache's predecessor - HugeCacheType hugeCache; - LargeCacheType largeCache; - - /* logical time, incremented on each put/get operation - To prevent starvation between pools, keep separately for each pool. - Overflow is OK, as we only want difference between - its current value and some recent. - - Both malloc and free should increment logical time, as in - a different case multiple cached blocks would have same age, - and accuracy of predictors suffers. - */ - uintptr_t cacheCurrTime; - - // memory pool that owns this LargeObjectCache, - ExtMemoryPool *extMemPool; // strict 1:1 relation, never changed - - static int sizeToIdx(size_t size); -public: - void init(ExtMemoryPool *memPool) { extMemPool = memPool; } - void put(LargeMemoryBlock *largeBlock); - void putList(LargeMemoryBlock *head); - LargeMemoryBlock *get(size_t size); - - void updateCacheState(DecreaseOrIncrease op, size_t size); - bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); - bool doCleanup(uintptr_t currTime, bool doThreshDecr); - - bool decreasingCleanup(); - bool regularCleanup(); - bool cleanAll(); - void reset() { - largeCache.reset(); - hugeCache.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif - static size_t alignToBin(size_t size) { - return size 0, ASSERT_TEXT); - suppress_unused_warning(prev); + intptr_t get() const { + MALLOC_ASSERT(setDone, ASSERT_TEXT); + return val; } - intptr_t getNumOfMods() const { return FencedLoad(binsModifications); } - // return true if need re-do the blocks search - inline bool waitTillBlockReleased(intptr_t startModifiedCnt); -}; -class CoalRequestQ { // queue of free blocks that coalescing was delayed -private: - FreeBlock *blocksToFree; - BackendSync *bkndSync; - // counted blocks in blocksToFree and that are leaved blocksToFree - // and still in active coalescing - intptr_t inFlyBlocks; -public: - void init(BackendSync *bSync) { bkndSync = bSync; } - FreeBlock *getAll(); // return current list of blocks and make queue empty - void putBlock(FreeBlock *fBlock); - inline void blockWasProcessed(); - intptr_t blocksInFly() const { return FencedLoad(inFlyBlocks); } -}; + // Note: set() can be called before init() + void set(intptr_t newVal) { + val = newVal; + setDone = true; + } -class MemExtendingSema { - intptr_t active; -public: - bool wait() { - bool rescanBins = false; - // up to 3 threads can add more memory from OS simultaneously, - // rest of threads have to wait - for (;;) { - intptr_t prevCnt = FencedLoad(active); - if (prevCnt < 3) { - intptr_t n = AtomicCompareExchange(active, prevCnt+1, prevCnt); - if (n == prevCnt) - break; - } else { - SpinWaitWhileEq(active, prevCnt); - rescanBins = true; - break; - } - } - return rescanBins; + bool ready() const { + return setDone; } - void signal() { AtomicAdd(active, -1); } -}; -enum MemRegionType { - // The region does not guarantee the block size. - MEMREG_FLEXIBLE_SIZE = 0, - // The region can hold exact number of blocks with the size of the - // first reqested block. - MEMREG_SEVERAL_BLOCKS, - // The region holds only one block with a reqested size. - MEMREG_ONE_BLOCK + // envName - environment variable to get controlled mode + void initReadEnv(const char *envName, intptr_t defaultVal) { + if (!setDone) { +#if !__TBB_WIN8UI_SUPPORT + // TODO: use strtol to get the actual value of the envirable + const char *envVal = getenv(envName); + if (envVal && !strcmp(envVal, "1")) + val = 1; + else +#endif + val = defaultVal; + setDone = true; + } + } }; -class MemRegionList { - MallocMutex regionListLock; -public: - MemRegion *head; - void add(MemRegion *r); - void remove(MemRegion *r); - int reportStat(FILE *f); +// Page type to be used inside MapMemory. +// Regular (4KB aligned), Huge and Transparent Huge Pages (2MB aligned). +enum PageType { + REGULAR = 0, + PREALLOCATED_HUGE_PAGE, + TRANSPARENT_HUGE_PAGE }; -class Backend { +// init() and printStatus() is called only under global initialization lock. +// Race is possible between registerAllocation() and registerReleasing(), +// harm is that up to single huge page releasing is missed (because failure +// to get huge page is registered only 1st time), that is negligible. +// setMode is also can be called concurrently. +// Object must reside in zero-initialized memory +// TODO: can we check for huge page presence during every 10th mmap() call +// in case huge page is released by another process? +class HugePagesStatus { private: -/* Blocks in range [minBinnedSize; getMaxBinnedSize()] are kept in bins, - one region can contains several blocks. Larger blocks are allocated directly - and one region always contains one block. -*/ - enum { - minBinnedSize = 8*1024UL, - /* If huge pages are available, maxBinned_HugePage used. - If not, maxBinned_SmallPage is the threshold. - TODO: use pool's granularity for upper bound setting.*/ - maxBinned_SmallPage = 1024*1024UL, - // TODO: support other page sizes - maxBinned_HugePage = 4*1024*1024UL - }; - enum { - VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result - }; -public: - static const int freeBinsNum = - (maxBinned_HugePage-minBinnedSize)/LargeObjectCache::largeBlockCacheStep + 1; - - // if previous access missed per-thread slabs pool, - // allocate numOfSlabAllocOnMiss blocks in advance - static const int numOfSlabAllocOnMiss = 2; - - enum { - NO_BIN = -1, - // special bin for blocks >= maxBinned_HugePage, blocks go to this bin - // when pool is created with keepAllMemory policy - // TODO: currently this bin is scanned using "1st fit", as it accumulates - // blocks of different sizes, "best fit" is preferred in terms of fragmentation - HUGE_BIN = freeBinsNum-1 - }; - - // Bin keeps 2-linked list of free blocks. It must be 2-linked - // because during coalescing a block it's removed from a middle of the list. - struct Bin { - FreeBlock *head, - *tail; - MallocMutex tLock; - - void removeBlock(FreeBlock *fBlock); - void reset() { head = tail = 0; } - bool empty() const { return !head; } - - size_t countFreeBlocks(); - size_t reportFreeBlocks(FILE *f); - void reportStat(FILE *f); - }; - - typedef BitMaskMin BitMaskBins; - - // array of bins supplemented with bitmask for fast finding of non-empty bins - class IndexedBins { - BitMaskBins bitMask; - Bin freeBins[Backend::freeBinsNum]; - FreeBlock *getFromBin(int binIdx, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, bool wait, - int *resLocked); - public: - FreeBlock *findBlock(int nativeBin, BackendSync *sync, size_t size, - bool resSlabAligned, bool alignedBin, int *numOfLockedBins); - bool tryReleaseRegions(int binIdx, Backend *backend); - void lockRemoveBlock(int binIdx, FreeBlock *fBlock); - void addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail); - bool tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail); - int getMinNonemptyBin(unsigned startBin) const { - int p = bitMask.getMinTrue(startBin); - return p == -1 ? Backend::freeBinsNum : p; + AllocControlledMode requestedMode; // changed only by user + // to keep enabled and requestedMode consistent + MallocMutex setModeLock; + size_t pageSize; + intptr_t needActualStatusPrint; + + static void doPrintStatus(bool state, const char *stateName) { + // Under macOS* fprintf/snprintf acquires an internal lock, so when + // 1st allocation is done under the lock, we got a deadlock. + // Do not use fprintf etc during initialization. + fputs("TBBmalloc: huge pages\t", stderr); + if (!state) + fputs("not ", stderr); + fputs(stateName, stderr); + fputs("\n", stderr); + } + + void parseSystemMemInfo() { + bool hpAvailable = false; + bool thpAvailable = false; + unsigned long long hugePageSize = 0; + +#if __linux__ + // Check huge pages existence + unsigned long long meminfoHugePagesTotal = 0; + + parseFileItem meminfoItems[] = { + // Parse system huge page size + { "Hugepagesize: %llu kB", hugePageSize }, + // Check if there are preallocated huge pages on the system + // https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt + { "HugePages_Total: %llu", meminfoHugePagesTotal } }; + + parseFile("/proc/meminfo", meminfoItems); + + // Double check another system information regarding preallocated + // huge pages if there are no information in /proc/meminfo + unsigned long long vmHugePagesTotal = 0; + + parseFileItem vmItem[] = { { "%llu", vmHugePagesTotal } }; + + // We parse a counter number, it can't be huge + parseFile("/proc/sys/vm/nr_hugepages", vmItem); + + if (meminfoHugePagesTotal > 0 || vmHugePagesTotal > 0) { + MALLOC_ASSERT(hugePageSize != 0, "Huge Page size can't be zero if we found preallocated."); + + // Any non zero value clearly states that there are preallocated + // huge pages on the system + hpAvailable = true; } - void verify(); - void reset(); - void reportStat(FILE *f); - }; -private: - class AdvRegionsBins { - BitMaskBins bins; - public: - void registerBin(int regBin) { bins.set(regBin, 1); } - int getMinUsedBin(int start) const { return bins.getMinTrue(start); } - void reset() { bins.reset(); } - }; - // auxiliary class to atomic maximum request finding - class MaxRequestComparator { - const Backend *backend; - public: - MaxRequestComparator(const Backend *be) : backend(be) {} - inline bool operator()(size_t oldMaxReq, size_t requestSize) const; - }; - -#if CHECK_ALLOCATION_RANGE - // Keep min and max of all addresses requested from OS, - // use it for checking memory possibly allocated by replaced allocators - // and for debugging purposes. Valid only for default memory pool. - class UsedAddressRange { - static const uintptr_t ADDRESS_UPPER_BOUND = UINTPTR_MAX; - - uintptr_t leftBound, - rightBound; - MallocMutex mutex; - public: - // rightBound is zero-initialized - void init() { leftBound = ADDRESS_UPPER_BOUND; } - void registerAlloc(uintptr_t left, uintptr_t right); - void registerFree(uintptr_t left, uintptr_t right); - // as only left and right bounds are kept, we can return true - // for pointer not allocated by us, if more than single region - // was requested from OS - bool inRange(void *ptr) const { - const uintptr_t p = (uintptr_t)ptr; - return leftBound<=p && p<=rightBound; + // Check if there is transparent huge pages support on the system + unsigned long long thpPresent = 'n'; + parseFileItem thpItem[] = { { "[alwa%cs] madvise never\n", thpPresent } }; + parseFile("/sys/kernel/mm/transparent_hugepage/enabled", thpItem); + + if (thpPresent == 'y') { + MALLOC_ASSERT(hugePageSize != 0, "Huge Page size can't be zero if we found thp existence."); + thpAvailable = true; } - }; -#else - class UsedAddressRange { - public: - void init() { } - void registerAlloc(uintptr_t, uintptr_t) {} - void registerFree(uintptr_t, uintptr_t) {} - bool inRange(void *) const { return true; } - }; #endif + MALLOC_ASSERT(!pageSize, "Huge page size can't be set twice. Double initialization."); - ExtMemoryPool *extMemPool; - // used for release every region on pool destroying - MemRegionList regionList; - - CoalRequestQ coalescQ; // queue of coalescing requests - BackendSync bkndSync; - // semaphore protecting adding more more memory from OS - MemExtendingSema memExtendingSema; - size_t totalMemSize, - memSoftLimit; - UsedAddressRange usedAddrRange; - // to keep 1st allocation large than requested, keep bootstrapping status - enum { - bootsrapMemNotDone = 0, - bootsrapMemInitializing, - bootsrapMemDone - }; - intptr_t bootsrapMemStatus; - MallocMutex bootsrapMemStatusMutex; - - // Using of maximal observed requested size allows decrease - // memory consumption for small requests and decrease fragmentation - // for workloads when small and large allocation requests are mixed. - // TODO: decrease, not only increase it - size_t maxRequestedSize; - - FreeBlock *addNewRegion(size_t size, MemRegionType type, bool addToBin); - FreeBlock *findBlockInRegion(MemRegion *region, size_t exactBlockSize); - void startUseBlock(MemRegion *region, FreeBlock *fBlock, bool addToBin); - void releaseRegion(MemRegion *region); - - FreeBlock *releaseMemInCaches(intptr_t startModifiedCnt, - int *lockedBinsThreshold, int numOfLockedBins); - void requestBootstrapMem(); - FreeBlock *askMemFromOS(size_t totalReqSize, intptr_t startModifiedCnt, - int *lockedBinsThreshold, int numOfLockedBins, - bool *splittable); - FreeBlock *genericGetBlock(int num, size_t size, bool resSlabAligned); - void genericPutBlock(FreeBlock *fBlock, size_t blockSz); - FreeBlock *splitUnalignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedRes); - FreeBlock *splitAlignedBlock(FreeBlock *fBlock, int num, size_t size, - bool needAlignedRes); - - FreeBlock *doCoalesc(FreeBlock *fBlock, MemRegion **memRegion); - bool coalescAndPutList(FreeBlock *head, bool forceCoalescQDrop, bool reportBlocksProcessed); - void coalescAndPut(FreeBlock *fBlock, size_t blockSz); - - void removeBlockFromBin(FreeBlock *fBlock); - - void *allocRawMem(size_t &size); - bool freeRawMem(void *object, size_t size); - - void putLargeBlock(LargeMemoryBlock *lmb); - void releaseCachesToLimit(); -public: - bool scanCoalescQ(bool forceCoalescQDrop); - intptr_t blocksInCoalescing() const { return coalescQ.blocksInFly(); } - void verify(); - void init(ExtMemoryPool *extMemoryPool); - void reset(); - bool destroy(); - bool clean(); // clean on caches cleanup - void reportStat(FILE *f); - - BlockI *getSlabBlock(int num) { - BlockI *b = (BlockI*) - genericGetBlock(num, slabSize, /*resSlabAligned=*/true); - MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT); - return b; + // Initialize object variables + pageSize = hugePageSize * 1024; // was read in KB from meminfo + isHPAvailable = hpAvailable; + isTHPAvailable = thpAvailable; } - void putSlabBlock(BlockI *block) { - genericPutBlock((FreeBlock *)block, slabSize); - } - void *getBackRefSpace(size_t size, bool *rawMemUsed); - void putBackRefSpace(void *b, size_t size, bool rawMemUsed); - bool inUserPool() const; +public: - LargeMemoryBlock *getLargeBlock(size_t size); - void returnLargeObject(LargeMemoryBlock *lmb); + // System information + bool isHPAvailable; + bool isTHPAvailable; - void *remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment); + // User defined value + bool isEnabled; - void setRecommendedMaxSize(size_t softLimit) { - memSoftLimit = softLimit; - releaseCachesToLimit(); + void init() { + parseSystemMemInfo(); + MallocMutex::scoped_lock lock(setModeLock); + requestedMode.initReadEnv("TBB_MALLOC_USE_HUGE_PAGES", 0); + isEnabled = (isHPAvailable || isTHPAvailable) && requestedMode.get(); } - inline size_t getMaxBinnedSize() const; - - bool ptrCanBeValid(void *ptr) const { return usedAddrRange.inRange(ptr); } - -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getTotalMemSize() const { return totalMemSize; } -#endif -private: - static int sizeToBin(size_t size) { - if (size >= maxBinned_HugePage) - return HUGE_BIN; - else if (size < minBinnedSize) - return NO_BIN; - int bin = (size - minBinnedSize)/LargeObjectCache::largeBlockCacheStep; + // Could be set from user code at any place. + // If we didn't call init() at this place, isEnabled will be false + void setMode(intptr_t newVal) { + MallocMutex::scoped_lock lock(setModeLock); + requestedMode.set(newVal); + isEnabled = (isHPAvailable || isTHPAvailable) && newVal; + } - MALLOC_ASSERT(bin < HUGE_BIN, "Invalid size."); - return bin; + bool isRequested() const { + return requestedMode.ready() ? requestedMode.get() : false; } -#if __TBB_MALLOC_BACKEND_STAT - static size_t binToSize(int bin) { - MALLOC_ASSERT(bin <= HUGE_BIN, "Invalid bin."); - return bin*LargeObjectCache::largeBlockCacheStep + minBinnedSize; + void reset() { + pageSize = needActualStatusPrint = 0; + isEnabled = isHPAvailable = isTHPAvailable = false; } -#endif - static bool toAlignedBin(FreeBlock *block, size_t size) { - return isAligned((char*)block+size, slabSize) - && size >= slabSize; + + // If memory mapping size is a multiple of huge page size, some OS kernels + // can use huge pages transparently. Use this when huge pages are requested. + size_t getGranularity() const { + if (requestedMode.ready()) + return requestedMode.get() ? pageSize : 0; + else + return HUGE_PAGE_SIZE; // the mode is not yet known; assume typical 2MB huge pages } - // register bins related to advance regions - AdvRegionsBins advRegBins; - IndexedBins freeLargeBins, - freeAlignedBins; + void printStatus() { + doPrintStatus(requestedMode.get(), "requested"); + if (requestedMode.get()) { // report actual status iff requested + if (pageSize) + FencedStore(needActualStatusPrint, 1); + else + doPrintStatus(/*state=*/false, "available"); + } + } }; class AllLargeBlocksList { @@ -963,86 +607,6 @@ struct FreeObject { FreeObject *next; }; -// An TBB allocator mode that can be controlled by user -// via API/environment variable. Must be placed in zero-initialized memory. -// External synchronization assumed. -// TODO: TBB_VERSION support -class AllocControlledMode { - intptr_t val; - bool setDone; -public: - bool ready() const { return setDone; } - intptr_t get() const { - MALLOC_ASSERT(setDone, ASSERT_TEXT); - return val; - } - void set(intptr_t newVal) { // note set() can be called before init() - val = newVal; - setDone = true; - } - // envName - environment variable to get controlled mode - void initReadEnv(const char *envName, intptr_t defaultVal); -}; - -// init() and printStatus() is called only under global initialization lock. -// Race is possible between registerAllocation() and registerReleasing(), -// harm is that up to single huge page releasing is missed (because failure -// to get huge page is registered only 1st time), that is negligible. -// setMode is also can be called concurrently. -// Object must reside in zero-initialized memory -// TODO: can we check for huge page presence during every 10th mmap() call -// in case huge page is released by another process? -class HugePagesStatus { -private: - AllocControlledMode requestedMode; // changed only by user - // to keep enabled and requestedMode consistent - MallocMutex setModeLock; - size_t pageSize; - intptr_t needActualStatusPrint; - - static void doPrintStatus(bool state, const char *stateName); -public: - // both variables are changed only inside HugePagesStatus - intptr_t enabled; - // Have we got huge pages at all? It's used when large hugepage-aligned - // region is releasing, to find can it release some huge pages or not. - intptr_t wasObserved; - - // If memory mapping size is a multiple of huge page size, some OS kernels - // can use huge pages transparently (i.e. even if not explicitly enabled). - // Use this when huge pages are requested. - size_t recommendedGranularity() const { - if (requestedMode.ready()) - return requestedMode.get()? pageSize : 0; - else - return 2048*1024; // the mode is not yet known; assume typical 2MB huge pages - } - void printStatus(); - void registerAllocation(bool available); - void registerReleasing(void* addr, size_t size); - - void init(size_t hugePageSize) { - MALLOC_ASSERT(!hugePageSize || isPowerOfTwo(hugePageSize), - "Only memory pages of a power-of-two size are supported."); - MALLOC_ASSERT(!pageSize, "Huge page size can't be set twice."); - pageSize = hugePageSize; - - MallocMutex::scoped_lock lock(setModeLock); - requestedMode.initReadEnv("TBB_MALLOC_USE_HUGE_PAGES", 0); - enabled = pageSize && requestedMode.get(); - } - void setMode(intptr_t newVal) { - MallocMutex::scoped_lock lock(setModeLock); - requestedMode.set(newVal); - enabled = pageSize && newVal; - } - void reset() { - pageSize = 0; - needActualStatusPrint = enabled = wasObserved = 0; - } -}; - -extern HugePagesStatus hugePages; /******* A helper class to support overriding malloc with scalable_malloc *******/ #if MALLOC_CHECK_RECURSION diff --git a/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h b/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h index 301c9f59..d3cfe29e 100644 --- a/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h +++ b/src/tbb/src/tbbmalloc/tbbmalloc_internal_api.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #ifndef __TBB_tbbmalloc_internal_api_H @@ -32,7 +28,7 @@ typedef enum { } AllocationModeInternalParam; void MallocInitializeITT(); -void __TBB_mallocProcessShutdownNotification(); +void __TBB_mallocProcessShutdownNotification(bool); #if _WIN32||_WIN64 void __TBB_mallocThreadShutdownNotification(); #endif diff --git a/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def index f2ca7f31..39237199 100644 --- a/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/win32-gcc-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -47,6 +43,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvj; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvjj; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEjj; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local:*; }; diff --git a/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def index f02c208b..9822ca0e 100644 --- a/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/win32-tbbmalloc-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS @@ -45,3 +41,5 @@ __TBB_malloc_safer_aligned_realloc ?pool_aligned_realloc@rml@@YAPAXPAVMemoryPool@1@PAXII@Z ?pool_aligned_malloc@rml@@YAPAXPAVMemoryPool@1@II@Z ?pool_identify@rml@@YAPAVMemoryPool@1@PAX@Z +?pool_msize@rml@@YAIPAVMemoryPool@1@PAX@Z + diff --git a/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def index d399b96d..c5400d4e 100644 --- a/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/win64-gcc-tbbmalloc-export.def @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ { @@ -47,6 +43,7 @@ _ZN3rml12pool_reallocEPNS_10MemoryPoolEPvy; _ZN3rml20pool_aligned_reallocEPNS_10MemoryPoolEPvyy; _ZN3rml19pool_aligned_mallocEPNS_10MemoryPoolEyy; _ZN3rml13pool_identifyEPv; +_ZN3rml10pool_msizeEPNS_10MemoryPoolEPv; local:*; }; diff --git a/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def b/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def index 0e335a49..33ee6e59 100644 --- a/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def +++ b/src/tbb/src/tbbmalloc/win64-tbbmalloc-export.def @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; EXPORTS @@ -46,3 +42,5 @@ __TBB_malloc_safer_aligned_realloc ?pool_aligned_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K2@Z ?pool_aligned_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K1@Z ?pool_identify@rml@@YAPEAVMemoryPool@1@PEAX@Z +?pool_msize@rml@@YA_KPEAVMemoryPool@1@PEAX@Z + diff --git a/src/tbb/src/tbbproxy/tbbproxy-windows.asm b/src/tbb/src/tbbproxy/tbbproxy-windows.asm index d5affed9..11b8dac5 100644 --- a/src/tbb/src/tbbproxy/tbbproxy-windows.asm +++ b/src/tbb/src/tbbproxy/tbbproxy-windows.asm @@ -1,4 +1,4 @@ -; Copyright (c) 2005-2017 Intel Corporation +; Copyright (c) 2005-2019 Intel Corporation ; ; Licensed under the Apache License, Version 2.0 (the "License"); ; you may not use this file except in compliance with the License. @@ -11,10 +11,6 @@ ; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ; See the License for the specific language governing permissions and ; limitations under the License. -; -; -; -; // __TBB_STRING macro defined in "tbb_stddef.h". However, we cannot include "tbb_stddef.h" // because it contains a lot of C/C++ definitions. So, we have to define __TBB_STRING here: diff --git a/src/tbb/src/tbbproxy/tbbproxy.cpp b/src/tbb/src/tbbproxy/tbbproxy.cpp index 0e57ab71..eba68bfa 100644 --- a/src/tbb/src/tbbproxy/tbbproxy.cpp +++ b/src/tbb/src/tbbproxy/tbbproxy.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2017 Intel Corporation + Copyright (c) 2005-2019 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,10 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - - - */ #include "tbb/tbb_config.h" @@ -23,6 +19,7 @@ #define TBB_PREVIEW_RUNTIME_LOADER 1 #include "tbb/runtime_loader.h" #include "tbb/tbb_stddef.h" +#include "tbb_environment.h" // C standard headers. #include // isspace @@ -109,8 +106,7 @@ static void _say( char const * format, va_list args ) { // To enable printing, the variable must be set and not empty. // Do not call it directly, use tell() instead. static void _tell( char const * format, va_list args ) { - char const * var = getenv( "TBB_VERSION" ); - if ( var != NULL && var[ 0 ] != 0 ) { + if ( tbb::internal::GetBoolEnvironmentVariable("TBB_VERSION") ) { _say( format, args ); } // if } // _tell @@ -352,7 +348,7 @@ static tbb::runtime_loader::error_code _load( char const * dll_name, int min_ver tbb::runtime_loader::error_code code = tbb::runtime_loader::ec_ok; /* - If these variables declared at the first usage, Intel C++ Compiler may issue warning(s): + If these variables declared at the first usage, Intel(R) C++ Compiler may issue warning(s): transfer of control [goto error] bypasses initialization of: ... Declaring variables at the beginning of the function eliminates warnings. */ diff --git a/src/tbb/src/test/harness.h b/src/tbb/src/test/harness.h new file mode 100644 index 00000000..ed47a360 --- /dev/null +++ b/src/tbb/src/test/harness.h @@ -0,0 +1,833 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for rock-bottom simple test harness. +// Just include this file to use it. +// Every test is presumed to have a command line of the form "test [-v] [MinThreads[:MaxThreads]]" +// The default for MinThreads is 1, for MaxThreads 4. +// The defaults can be overridden by defining macros HARNESS_DEFAULT_MIN_THREADS +// and HARNESS_DEFAULT_MAX_THREADS before including harness.h + +#ifndef tbb_tests_harness_H +#define tbb_tests_harness_H + +#include "tbb/tbb_config.h" +#include "harness_defs.h" + +namespace Harness { + enum TestResult { + Done, + Skipped, + Unknown + }; +} + +//! Entry point to a TBB unit test application +/** It MUST be defined by the test application. + + If HARNESS_NO_PARSE_COMMAND_LINE macro was not explicitly set before including harness.h, + then global variables MinThread, and MaxThread will be available and + initialized when it is called. + + Returns Harness::Done when the tests passed successfully. When the test fail, it must + not return, calling exit(errcode) or abort() instead. When the test is not supported + for the given platform/compiler/etc, it should return Harness::Skipped. + + To provide non-standard variant of main() for the test, define HARNESS_CUSTOM_MAIN + before including harness.h **/ +int TestMain (); + +#if __SUNPRO_CC + #include + #include + #include +#else /* !__SUNPRO_CC */ + #include + #include +#endif /* !__SUNPRO_CC */ + +#include + +#if __TBB_MIC_NATIVE + #include "harness_mic.h" +#else + #define HARNESS_EXPORT + #define REPORT_FATAL_ERROR REPORT +#endif /* !__MIC__ */ + +#if _WIN32||_WIN64 + #include "tbb/machine/windows_api.h" + #if _WIN32_WINNT > 0x0501 && _MSC_VER && !_M_ARM + // Suppress "typedef ignored ... when no variable is declared" warning by vc14 + #pragma warning (push) + #pragma warning (disable: 4091) + #include + #pragma warning (pop) + #pragma comment (lib, "dbghelp.lib") + #endif + #if __TBB_WIN8UI_SUPPORT + #include + #endif + #if _MSC_VER + #include + #endif + #include +#else + #include +#endif + +#if __linux__ + #include /* for uname */ + #include /* for use in LinuxKernelVersion() */ + #include +#endif +// at least GLIBC 2.1 or OSX 10.5 +#if __GLIBC__>2 || ( __GLIBC__==2 && __GLIBC_MINOR__ >= 1) || __APPLE__ + #include /*backtrace*/ + #define BACKTRACE_FUNCTION_AVAILABLE 1 +#endif + +namespace Harness { + class NativeMutex { +#if _WIN32||_WIN64 + CRITICAL_SECTION my_critical_section; + public: + NativeMutex() { + InitializeCriticalSectionEx(&my_critical_section, 4000, 0); + } + void lock() { + EnterCriticalSection(&my_critical_section); + } + void unlock() { + LeaveCriticalSection(&my_critical_section); + } + ~NativeMutex() { + DeleteCriticalSection(&my_critical_section); + } +#else + pthread_mutex_t m_mutex; + public: + NativeMutex() { + pthread_mutex_init(&m_mutex, NULL); + } + void lock() { + pthread_mutex_lock(&m_mutex); + } + void unlock() { + pthread_mutex_unlock(&m_mutex); + } + ~NativeMutex() { + pthread_mutex_destroy(&m_mutex); + } +#endif + }; + namespace internal { + static NativeMutex print_stack_mutex; + } +} + +#include "harness_runtime_loader.h" +#include "harness_report.h" + +//! Prints current call stack +void print_call_stack() { + Harness::internal::print_stack_mutex.lock(); + fflush(stdout); fflush(stderr); + #if BACKTRACE_FUNCTION_AVAILABLE + const int sz = 100; // max number of frames to capture + void *buff[sz]; + int n = backtrace(buff, sz); + REPORT("Call stack info (%d):\n", n); + backtrace_symbols_fd(buff, n, fileno(stdout)); + #elif __SUNPRO_CC + REPORT("Call stack info:\n"); + printstack(fileno(stdout)); + #elif _WIN32_WINNT > 0x0501 && _MSC_VER>=1500 && !__TBB_WIN8UI_SUPPORT + const int sz = 62; // XP limitation for number of frames + void *buff[sz]; + int n = CaptureStackBackTrace(0, sz, buff, NULL); + REPORT("Call stack info (%d):\n", n); + static LONG once = 0; + if( !InterlockedExchange(&once, 1) ) + SymInitialize(GetCurrentProcess(), NULL, TRUE); + const int len = 255; // just some reasonable string buffer size + union { SYMBOL_INFO sym; char pad[sizeof(SYMBOL_INFO)+len]; }; + sym.MaxNameLen = len; + sym.SizeOfStruct = sizeof( SYMBOL_INFO ); + DWORD64 offset; + for(int i = 1; i < n; i++) { // skip current frame + if(!SymFromAddr( GetCurrentProcess(), DWORD64(buff[i]), &offset, &sym )) { + sym.Address = ULONG64(buff[i]); offset = 0; sym.Name[0] = 0; + } + REPORT("[%d] %016I64X+%04I64X: %s\n", i, sym.Address, offset, sym.Name); //TODO: print module name + } + #endif /*BACKTRACE_FUNCTION_AVAILABLE*/ + Harness::internal::print_stack_mutex.unlock(); +} + +#if !HARNESS_NO_ASSERT + #include //for set_terminate + #include "harness_assert.h" + #if TEST_USES_TBB + #include /*set_assertion_handler*/ + #endif + + struct InitReporter { + void (*default_terminate_handler)() ; + InitReporter(): default_terminate_handler(NULL) { + #if TEST_USES_TBB + #if TBB_USE_ASSERT + tbb::set_assertion_handler(ReportError); + #endif + ASSERT_WARNING(TBB_INTERFACE_VERSION <= tbb::TBB_runtime_interface_version(), "runtime version mismatch"); + #endif + #if TBB_USE_EXCEPTIONS + default_terminate_handler = std::set_terminate(handle_terminate); + #endif + } + static void handle_terminate(); + }; + static InitReporter InitReportError; + + void InitReporter::handle_terminate(){ + REPORT("std::terminate called.\n"); + print_call_stack(); + if (InitReportError.default_terminate_handler){ + InitReportError.default_terminate_handler(); + } + } + + typedef void (*test_error_extra_t)(void); + static test_error_extra_t ErrorExtraCall; + //! Set additional handler to process failed assertions + void SetHarnessErrorProcessing( test_error_extra_t extra_call ) { + ErrorExtraCall = extra_call; + } + + //! Reports errors issued by failed assertions + void ReportError( const char* filename, int line, const char* expression, const char * message ) { + print_call_stack(); + #if __TBB_ICL_11_1_CODE_GEN_BROKEN + printf("%s:%d, assertion %s: %s\n", filename, line, expression, message ? message : "failed" ); + #else + REPORT_FATAL_ERROR("%s:%d, assertion %s: %s\n", filename, line, expression, message ? message : "failed" ); + #endif + + if( ErrorExtraCall ) + (*ErrorExtraCall)(); + fflush(stdout); fflush(stderr); + #if HARNESS_TERMINATE_ON_ASSERT + TerminateProcess(GetCurrentProcess(), 1); + #elif HARNESS_EXIT_ON_ASSERT + exit(1); + #elif HARNESS_CONTINUE_ON_ASSERT + // continue testing + #elif _MSC_VER && _DEBUG + // aligned with tbb_assert_impl.h behavior + if(1 == _CrtDbgReport(_CRT_ASSERT, filename, line, NULL, "%s\r\n%s", expression, message?message:"")) + _CrtDbgBreak(); + #else + abort(); + #endif /* HARNESS_EXIT_ON_ASSERT */ + } + //! Reports warnings issued by failed warning assertions + void ReportWarning( const char* filename, int line, const char* expression, const char * message ) { + REPORT("Warning: %s:%d, assertion %s: %s\n", filename, line, expression, message ? message : "failed" ); + } + +#else /* !HARNESS_NO_ASSERT */ + + #define ASSERT(p,msg) (Harness::suppress_unused_warning(p), (void)0) + #define ASSERT_WARNING(p,msg) (Harness::suppress_unused_warning(p), (void)0) + +#endif /* !HARNESS_NO_ASSERT */ + +namespace Harness { + //TODO: unify with utility::internal::array_length from examples common utilities + template + inline size_t array_length(const T(&)[N]) + { + return N; + } + + template + inline T* end( T(& array)[N]) + { + return array+ array_length(array) ; + } + +} //namespace Harness + +#if TEST_USES_TBB + #include "tbb/blocked_range.h" + + namespace Harness { + template + tbb::blocked_range make_blocked_range( T(& array)[N]){ return tbb::blocked_range(array, array + N);} + } +#endif + +#if !HARNESS_NO_PARSE_COMMAND_LINE + +//! Controls level of commentary printed via printf-like REMARK() macro. +/** If true, makes the test print commentary. If false, test should print "done" and nothing more. */ +static bool Verbose; + +#ifndef HARNESS_DEFAULT_MIN_THREADS + #define HARNESS_DEFAULT_MIN_THREADS 1 +#endif + +//! Minimum number of threads +static int MinThread = HARNESS_DEFAULT_MIN_THREADS; + +#ifndef HARNESS_DEFAULT_MAX_THREADS + #define HARNESS_DEFAULT_MAX_THREADS 4 +#endif + +//! Maximum number of threads +static int MaxThread = HARNESS_DEFAULT_MAX_THREADS; + +//! Parse command line of the form "name [-v] [MinThreads[:MaxThreads]]" +/** Sets Verbose, MinThread, and MaxThread accordingly. + The nthread argument can be a single number or a range of the form m:n. + A single number m is interpreted as if written m:m. + The numbers must be non-negative. + Clients often treat the value 0 as "run sequentially." */ +inline void ParseCommandLine( int argc, char* argv[] ) { + if( !argc ) REPORT("Command line with 0 arguments\n"); + int i = 1; + if( i + +//! Base class for objects which support move ctors +class Movable { +public: + Movable() : alive(true) {} + void Reset() { alive = true; } + Movable(Movable&& other) { + ASSERT(other.alive, "Moving from a dead object"); + alive = true; + other.alive = false; + } + Movable& operator=(Movable&& other) { + ASSERT(alive, "Assignment to a dead object"); + ASSERT(other.alive, "Assignment of a dead object"); + other.alive = false; + return *this; + } + Movable& operator=(const Movable& other) { + ASSERT(alive, "Assignment to a dead object"); + ASSERT(other.alive, "Assignment of a dead object"); + return *this; + } + Movable(const Movable& other) { + ASSERT(other.alive, "Const reference to a dead object"); + alive = true; + } + ~Movable() { alive = false; } + volatile bool alive; +}; + +class MoveOnly : Movable, NoCopy { +public: + MoveOnly() : Movable() {} + MoveOnly(MoveOnly&& other) : Movable( std::move(other) ) {} +}; +#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ + +#if HARNESS_TBBMALLOC_THREAD_SHUTDOWN && __TBB_SOURCE_DIRECTLY_INCLUDED && (_WIN32||_WIN64) +#include "../tbbmalloc/tbbmalloc_internal_api.h" +#endif + +//! For internal use by template function NativeParallelFor +template +class NativeParallelForTask: NoCopy { +public: + NativeParallelForTask( Index index_, const Body& body_ ) : + index(index_), + body(body_) + {} + + //! Start task + void start() { +#if _WIN32||_WIN64 + unsigned thread_id; +#if __TBB_WIN8UI_SUPPORT + std::thread* thread_tmp=new std::thread(thread_function, this); + thread_handle = thread_tmp->native_handle(); + thread_id = 0; +#else + unsigned stack_size = 0; +#if HARNESS_THREAD_STACK_SIZE + stack_size = HARNESS_THREAD_STACK_SIZE; +#endif + thread_handle = (HANDLE)_beginthreadex( NULL, stack_size, thread_function, this, 0, &thread_id ); +#endif + ASSERT( thread_handle!=0, "NativeParallelFor: _beginthreadex failed" ); +#else +#if __ICC==1100 + #pragma warning (push) + #pragma warning (disable: 2193) +#endif /* __ICC==1100 */ + // Some machines may have very large hard stack limit. When the test is + // launched by make, the default stack size is set to the hard limit, and + // calls to pthread_create fail with out-of-memory error. + // Therefore we set the stack size explicitly (as for TBB worker threads). +#if !defined(HARNESS_THREAD_STACK_SIZE) +#if __i386__||__i386||__arm__ + const size_t stack_size = 1*MByte; +#elif __x86_64__ + const size_t stack_size = 2*MByte; +#else + const size_t stack_size = 4*MByte; +#endif +#else + const size_t stack_size = HARNESS_THREAD_STACK_SIZE; +#endif /* HARNESS_THREAD_STACK_SIZE */ + pthread_attr_t attr_stack; + int status = pthread_attr_init(&attr_stack); + ASSERT(0==status, "NativeParallelFor: pthread_attr_init failed"); + status = pthread_attr_setstacksize( &attr_stack, stack_size ); + ASSERT(0==status, "NativeParallelFor: pthread_attr_setstacksize failed"); + status = pthread_create(&thread_id, &attr_stack, thread_function, this); + ASSERT(0==status, "NativeParallelFor: pthread_create failed"); + pthread_attr_destroy(&attr_stack); +#if __ICC==1100 + #pragma warning (pop) +#endif +#endif /* _WIN32||_WIN64 */ + } + + //! Wait for task to finish + void wait_to_finish() { +#if _WIN32||_WIN64 + DWORD status = WaitForSingleObjectEx( thread_handle, INFINITE, FALSE ); + ASSERT( status!=WAIT_FAILED, "WaitForSingleObject failed" ); + CloseHandle( thread_handle ); +#else + int status = pthread_join( thread_id, NULL ); + ASSERT( !status, "pthread_join failed" ); +#endif +#if HARNESS_NO_ASSERT + (void)status; +#endif + } + +private: +#if _WIN32||_WIN64 + HANDLE thread_handle; +#else + pthread_t thread_id; +#endif + + //! Range over which task will invoke the body. + const Index index; + + //! Body to invoke over the range. + const Body body; + +#if _WIN32||_WIN64 + static unsigned __stdcall thread_function( void* object ) +#else + static void* thread_function(void* object) +#endif + { + NativeParallelForTask& self = *static_cast(object); + (self.body)(self.index); +#if HARNESS_TBBMALLOC_THREAD_SHUTDOWN && __TBB_SOURCE_DIRECTLY_INCLUDED && (_WIN32||_WIN64) + // in those cases can't release per-thread cache automatically, + // so do it manually + // TODO: investigate less-intrusive way to do it, for example via FLS keys + __TBB_mallocThreadShutdownNotification(); +#endif + return 0; + } +}; + +//! Execute body(i) in parallel for i in the interval [0,n). +/** Each iteration is performed by a separate thread. */ +template +void NativeParallelFor( Index n, const Body& body ) { + typedef NativeParallelForTask task; + + if( n>0 ) { + // Allocate array to hold the tasks + task* array = static_cast(operator new( n*sizeof(task) )); + + // Construct the tasks + for( Index i=0; i!=n; ++i ) + new( &array[i] ) task(i,body); + + // Start the tasks + for( Index i=0; i!=n; ++i ) + array[i].start(); + + // Wait for the tasks to finish and destroy each one. + for( Index i=n; i; --i ) { + array[i-1].wait_to_finish(); + array[i-1].~task(); + } + + // Deallocate the task array + operator delete(array); + } +} + +//! The function to zero-initialize arrays; useful to avoid warnings +template +void zero_fill(void* array, size_t n) { + memset(array, 0, sizeof(T)*n); +} + +#if __SUNPRO_CC && defined(min) +#undef min +#undef max +#endif + +#ifndef min +//! Utility template function returning lesser of the two values. +/** Provided here to avoid including not strict safe .\n + In case operands cause signed/unsigned or size mismatch warnings it is caller's + responsibility to do the appropriate cast before calling the function. **/ +template +T1 min ( const T1& val1, const T2& val2 ) { + return val1 < val2 ? val1 : val2; +} +#endif /* !min */ + +#ifndef max +//! Utility template function returning greater of the two values. +/** Provided here to avoid including not strict safe .\n + In case operands cause signed/unsigned or size mismatch warnings it is caller's + responsibility to do the appropriate cast before calling the function. **/ +template +T1 max ( const T1& val1, const T2& val2 ) { + return val1 < val2 ? val2 : val1; +} +#endif /* !max */ + +template +static inline bool is_aligned(T arg, size_t alignment) { + return 0==((size_t)arg & (alignment-1)); +} + +#if __linux__ +inline unsigned LinuxKernelVersion() +{ + unsigned digit1, digit2, digit3; + struct utsname utsnameBuf; + + if (-1 == uname(&utsnameBuf)) { + REPORT_FATAL_ERROR("Can't call uname: errno %d\n", errno); + exit(1); + } + if (3 != sscanf(utsnameBuf.release, "%u.%u.%u", &digit1, &digit2, &digit3)) { + REPORT_FATAL_ERROR("Unable to parse OS release '%s'\n", utsnameBuf.release); + exit(1); + } + return 1000000*digit1+1000*digit2+digit3; +} +#endif + +namespace Harness { + +#if !HARNESS_NO_ASSERT +//! Base class that asserts that no operations are made with the object after its destruction. +class NoAfterlife { +protected: + enum state_t { + LIVE=0x56781234, + DEAD=0xDEADBEEF + } m_state; + +public: + NoAfterlife() : m_state(LIVE) {} + NoAfterlife( const NoAfterlife& src ) : m_state(LIVE) { + ASSERT( src.IsLive(), "Constructing from the dead source" ); + } + ~NoAfterlife() { + ASSERT( IsLive(), "Repeated destructor call" ); + m_state = DEAD; + } + const NoAfterlife& operator=( const NoAfterlife& src ) { + ASSERT( IsLive(), NULL ); + ASSERT( src.IsLive(), NULL ); + return *this; + } + void AssertLive() const { + ASSERT( IsLive(), "Already dead" ); + } + bool IsLive() const { + return m_state == LIVE; + } +}; // NoAfterlife +#endif /* !HARNESS_NO_ASSERT */ + +#if _WIN32 || _WIN64 + void Sleep ( int ms ) { +#if !__TBB_WIN8UI_SUPPORT + ::Sleep(ms); +#else + std::chrono::milliseconds sleep_time( ms ); + std::this_thread::sleep_for( sleep_time ); +#endif + + } + + typedef DWORD tid_t; + tid_t CurrentTid () { return GetCurrentThreadId(); } + +#else /* !WIN */ + + void Sleep ( int ms ) { + timespec requested = { ms / 1000, (ms % 1000)*1000000 }; + timespec remaining = { 0, 0 }; + nanosleep(&requested, &remaining); + } + + typedef pthread_t tid_t; + tid_t CurrentTid () { return pthread_self(); } +#endif /* !WIN */ + + static const unsigned Primes[] = { + 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 0xe1626741, 0x79695e6b, + 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, + 0xbe4d6fe9, 0x5f15e201, 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, + 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 0x3d9910ed, 0x2e687b5b, + 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, + 0x54581edb, 0xf2480f45, 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, + 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 0xb892d829, 0x3549366b, + 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f + }; + + class FastRandom { + unsigned x, a; + public: + unsigned short get() { + unsigned short r = (unsigned short)(x >> 16); + x = x*a + 1; + return r; + } + explicit FastRandom( unsigned seed ) { + x = seed; + a = Primes[seed % (sizeof(Primes) / sizeof(Primes[0]))]; + } + }; + template + class FastRandomBody { + FastRandom r; + public: + explicit FastRandomBody( unsigned seed ) : r(seed) {} + // Depending on the input type T the result distribution formed from this operator() + // might possess different characteristics than the original one used in FastRandom instance. + T operator()() { return T(r.get()); } + }; + + int SetEnv( const char *envname, const char *envval ) { + ASSERT( envname && envval, "Harness::SetEnv() requires two valid C strings" ); +#if __TBB_WIN8UI_SUPPORT + ASSERT( false, "Harness::SetEnv() should not be called in code built for win8ui" ); + return -1; +#elif !(_MSC_VER || __MINGW32__ || __MINGW64__) + // On POSIX systems use setenv + return setenv(envname, envval, /*overwrite=*/1); +#elif __STDC_SECURE_LIB__>=200411 + // this macro is set in VC & MinGW if secure API functions are present + return _putenv_s(envname, envval); +#else + // If no secure API on Windows, use _putenv + size_t namelen = strlen(envname), valuelen = strlen(envval); + char* buf = new char[namelen+valuelen+2]; + strncpy(buf, envname, namelen); + buf[namelen] = '='; + strncpy(buf+namelen+1, envval, valuelen); + buf[namelen+1+valuelen] = char(0); + int status = _putenv(buf); + delete[] buf; + return status; +#endif + } + + char* GetEnv(const char *envname) { + ASSERT(envname, "Harness::GetEnv() requires a valid C string"); +#if __TBB_WIN8UI_SUPPORT + return NULL; +#else + return std::getenv(envname); +#endif + } + + class DummyBody { + int m_numIters; + public: + explicit DummyBody( int iters ) : m_numIters( iters ) {} + void operator()( int ) const { + for ( volatile int i = 0; i < m_numIters; ++i ) {} + } + }; +} // namespace Harness + +#endif /* tbb_tests_harness_H */ diff --git a/src/tbb/src/test/harness_allocator.h b/src/tbb/src/test/harness_allocator.h new file mode 100644 index 00000000..93e388cb --- /dev/null +++ b/src/tbb/src/test/harness_allocator.h @@ -0,0 +1,869 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for simple estimate of the memory being used by a program. +// Not yet implemented for macOS*. +// This header is an optional part of the test harness. +// It assumes that "harness_assert.h" has already been included. + +#ifndef tbb_test_harness_allocator_H +#define tbb_test_harness_allocator_H + +#include "harness_defs.h" + +#if __linux__ || __APPLE__ || __sun +#include +#elif _WIN32 +#include "tbb/machine/windows_api.h" +#endif /* OS specific */ +#include +#include +#include +#include +#include +#include __TBB_STD_SWAP_HEADER + +#include "tbb/atomic.h" +#include "tbb/tbb_allocator.h" + +#if __SUNPRO_CC +using std::printf; +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Workaround for overzealous compiler warnings in /Wp64 mode + #pragma warning (push) +#if defined(_Wp64) + #pragma warning (disable: 4267) +#endif +#if _MSC_VER <= 1600 + #pragma warning (disable: 4355) +#endif +#if _MSC_VER <= 1800 + #pragma warning (disable: 4512) +#endif +#endif + +#if TBB_INTERFACE_VERSION >= 7005 +// Allocator traits were introduced in 4.2 U5 +namespace Harness { +#if __TBB_ALLOCATOR_TRAITS_PRESENT + using std::true_type; + using std::false_type; +#else + using tbb::internal::true_type; + using tbb::internal::false_type; +#endif //__TBB_ALLOCATOR_TRAITS_PRESENT +} +#endif + +template +struct arena_data { + char * const my_buffer; + size_t const my_size; //in bytes + counter_type my_allocated; // in bytes + + template + arena_data(T * a_buffer, size_t a_size) __TBB_NOEXCEPT(true) + : my_buffer(reinterpret_cast(a_buffer)) + , my_size(a_size * sizeof(T)) + { + my_allocated =0; + } +private: + void operator=( const arena_data& ); // NoAssign is not used to avoid dependency on harness.h +}; + +template +struct arena { + typedef arena_data arena_data_t; +private: + arena_data_t * my_data; +public: + typedef T value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + template struct rebind { + typedef arena other; + }; + + typedef pocma propagate_on_container_move_assignment; + + arena(arena_data_t & data) __TBB_NOEXCEPT(true) : my_data(&data) {} + + template + friend struct arena; + + template + arena(arena const& other) __TBB_NOEXCEPT(true) : my_data(other.my_data) {} + + friend void swap(arena & lhs ,arena & rhs){ + std::swap(lhs.my_data, rhs.my_data); + } + + pointer address(reference x) const {return &x;} + const_pointer address(const_reference x) const {return &x;} + + //! Allocate space for n objects, starting on a cache/sector line. + pointer allocate( size_type n, const void* =0) { + size_t new_size = (my_data->my_allocated += n*sizeof(T)); + ASSERT(my_data->my_allocated <= my_data->my_size,"trying to allocate more than was reserved"); + char* result = &(my_data->my_buffer[new_size - n*sizeof(T)]); + return reinterpret_cast(result); + } + + //! Free block of memory that starts on a cache line + void deallocate( pointer p_arg, size_type n) { + char* p = reinterpret_cast(p_arg); + ASSERT(p >=my_data->my_buffer && p <= my_data->my_buffer + my_data->my_size, "trying to deallocate pointer not from arena ?"); + ASSERT(p + n*sizeof(T) <= my_data->my_buffer + my_data->my_size, "trying to deallocate incorrect number of items?"); + tbb::internal::suppress_unused_warning(p, n); + } + + //! Largest value for which method allocate might succeed. + size_type max_size() const throw() { + return my_data->my_size / sizeof(T); + } + + //! Copy-construct value at location pointed to by p. +#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + template + void construct(U *p, Args&&... args) + { ::new((void *)p) U(std::forward(args)...); } +#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC +#if __TBB_CPP11_RVALUE_REF_PRESENT + void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} +#endif + void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} +#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + + //! Destroy value at location pointed to by p. + void destroy( pointer p ) { + p->~value_type(); + // suppress "unreferenced parameter" warnings by MSVC up to and including 2015 + tbb::internal::suppress_unused_warning(p); + } + + friend bool operator==(arena const& lhs, arena const& rhs){ + return lhs.my_data == rhs.my_data; + } + + friend bool operator!=(arena const& lhs, arena const& rhs){ + return !(lhs== rhs); + } +}; + +template > +struct allocator_counters { + count_t items_allocated; + count_t items_freed; + count_t allocations; + count_t frees; + + friend bool operator==(allocator_counters const & lhs, allocator_counters const & rhs){ + return lhs.items_allocated == rhs.items_allocated + && lhs.items_freed == rhs.items_freed + && lhs.allocations == rhs.allocations + && lhs.frees == rhs.frees + ; + } +}; + +template > +class static_counting_allocator : public base_alloc_t +{ +public: + typedef typename base_alloc_t::pointer pointer; + typedef typename base_alloc_t::const_pointer const_pointer; + typedef typename base_alloc_t::reference reference; + typedef typename base_alloc_t::const_reference const_reference; + typedef typename base_alloc_t::value_type value_type; + typedef typename base_alloc_t::size_type size_type; + typedef typename base_alloc_t::difference_type difference_type; + template struct rebind { + typedef static_counting_allocator::other,count_t> other; + }; + + typedef allocator_counters counters_t; + + static size_t max_items; + static count_t items_allocated; + static count_t items_freed; + static count_t allocations; + static count_t frees; + static bool verbose, throwing; + + static_counting_allocator() throw() { } + + static_counting_allocator(const base_alloc_t& src) throw() + : base_alloc_t(src) { } + + static_counting_allocator(const static_counting_allocator& src) throw() + : base_alloc_t(src) { } + + template + static_counting_allocator(const static_counting_allocator& src) throw() + : base_alloc_t(src) { } + + pointer allocate(const size_type n) + { + if(verbose) printf("\t+%d|", int(n)); + if(max_items && items_allocated + n >= max_items) { + if(verbose) printf("items limit hits!"); + if(throwing) + __TBB_THROW( std::bad_alloc() ); + return NULL; + } + pointer p = base_alloc_t::allocate(n, pointer(0)); + allocations++; + items_allocated += n; + return p; + } + + pointer allocate(const size_type n, const void * const) + { return allocate(n); } + + void deallocate(const pointer ptr, const size_type n) + { + if(verbose) printf("\t-%d|", int(n)); + frees++; + items_freed += n; + base_alloc_t::deallocate(ptr, n); + } + + static counters_t counters(){ + counters_t c = {items_allocated, items_freed, allocations, frees} ; + return c; + } + + static void init_counters(bool v = false) { + verbose = v; + if(verbose) printf("\n------------------------------------------- Allocations:\n"); + items_allocated = 0; + items_freed = 0; + allocations = 0; + frees = 0; + max_items = 0; + } + + static void set_limits(size_type max = 0, bool do_throw = true) { + max_items = max; + throwing = do_throw; + } +}; + +template +size_t static_counting_allocator::max_items; +template +count_t static_counting_allocator::items_allocated; +template +count_t static_counting_allocator::items_freed; +template +count_t static_counting_allocator::allocations; +template +count_t static_counting_allocator::frees; +template +bool static_counting_allocator::verbose; +template +bool static_counting_allocator::throwing; + + +template > +class static_shared_counting_allocator_base +{ +public: + typedef allocator_counters counters_t; + + static size_t max_items; + static count_t items_allocated; + static count_t items_freed; + static count_t allocations; + static count_t frees; + static bool verbose, throwing; + + static counters_t counters(){ + counters_t c = {items_allocated, items_freed, allocations, frees} ; + return c; + } + + static void init_counters(bool v = false) { + verbose = v; + if(verbose) printf("\n------------------------------------------- Allocations:\n"); + items_allocated = 0; + items_freed = 0; + allocations = 0; + frees = 0; + max_items = 0; + } + + static void set_limits(size_t max = 0, bool do_throw = true) { + max_items = max; + throwing = do_throw; + } +}; + +template +size_t static_shared_counting_allocator_base::max_items; + +template +count_t static_shared_counting_allocator_base::items_allocated; + +template +count_t static_shared_counting_allocator_base::items_freed; + +template +count_t static_shared_counting_allocator_base::allocations; + +template +count_t static_shared_counting_allocator_base::frees; + +template +bool static_shared_counting_allocator_base::verbose; + +template +bool static_shared_counting_allocator_base::throwing; + +template > +class static_shared_counting_allocator : public static_shared_counting_allocator_base, public base_alloc_t +{ + typedef static_shared_counting_allocator_base base_t; +public: + typedef typename base_alloc_t::pointer pointer; + typedef typename base_alloc_t::const_pointer const_pointer; + typedef typename base_alloc_t::reference reference; + typedef typename base_alloc_t::const_reference const_reference; + typedef typename base_alloc_t::value_type value_type; + typedef typename base_alloc_t::size_type size_type; + typedef typename base_alloc_t::difference_type difference_type; + template struct rebind { + typedef static_shared_counting_allocator::other, count_t> other; + }; + + static_shared_counting_allocator() throw() { } + + static_shared_counting_allocator(const base_alloc_t& src) throw() + : base_alloc_t(src) { } + + static_shared_counting_allocator(const static_shared_counting_allocator& src) throw() + : base_alloc_t(src) { } + + template + static_shared_counting_allocator(const static_shared_counting_allocator& src) throw() + : base_alloc_t(src) { } + + pointer allocate(const size_type n) + { + if(base_t::verbose) printf("\t+%d|", int(n)); + if(base_t::max_items && base_t::items_allocated + n >= base_t::max_items) { + if(base_t::verbose) printf("items limit hits!"); + if(base_t::throwing) + __TBB_THROW( std::bad_alloc() ); + return NULL; + } + base_t::allocations++; + base_t::items_allocated += n; + return base_alloc_t::allocate(n, pointer(0)); + } + + pointer allocate(const size_type n, const void * const) + { return allocate(n); } + + void deallocate(const pointer ptr, const size_type n) + { + if(base_t::verbose) printf("\t-%d|", int(n)); + base_t::frees++; + base_t::items_freed += n; + base_alloc_t::deallocate(ptr, n); + } +}; + +template > +class local_counting_allocator : public base_alloc_t +{ +public: + typedef typename base_alloc_t::pointer pointer; + typedef typename base_alloc_t::const_pointer const_pointer; + typedef typename base_alloc_t::reference reference; + typedef typename base_alloc_t::const_reference const_reference; + typedef typename base_alloc_t::value_type value_type; + typedef typename base_alloc_t::size_type size_type; + typedef typename base_alloc_t::difference_type difference_type; + template struct rebind { + typedef local_counting_allocator::other,count_t> other; + }; + + count_t items_allocated; + count_t items_freed; + count_t allocations; + count_t frees; + size_t max_items; + + void set_counters(const count_t & a_items_allocated, const count_t & a_items_freed, const count_t & a_allocations, const count_t & a_frees, const count_t & a_max_items){ + items_allocated = a_items_allocated; + items_freed = a_items_freed; + allocations = a_allocations; + frees = a_frees; + max_items = a_max_items; + } + + template< typename allocator_t> + void set_counters(const allocator_t & a){ + this->set_counters(a.items_allocated, a.items_freed, a.allocations, a.frees, a.max_items); + } + + void clear_counters(){ + count_t zero; + zero = 0; + this->set_counters(zero,zero,zero,zero,zero); + } + + local_counting_allocator() throw() { + this->clear_counters(); + } + + local_counting_allocator(const local_counting_allocator &a) throw() + : base_alloc_t(a) + , items_allocated(a.items_allocated) + , items_freed(a.items_freed) + , allocations(a.allocations) + , frees(a.frees) + , max_items(a.max_items) + { } + + template + local_counting_allocator(const static_counting_allocator & a) throw() { + this->set_counters(a); + } + + template + local_counting_allocator(const local_counting_allocator &a) throw() + : items_allocated(a.items_allocated) + , items_freed(a.items_freed) + , allocations(a.allocations) + , frees(a.frees) + , max_items(a.max_items) + { } + + bool operator==(const local_counting_allocator &a) const + { return static_cast(a) == *this; } + + pointer allocate(const size_type n) + { + if(max_items && items_allocated + n >= max_items) + __TBB_THROW( std::bad_alloc() ); + pointer p = base_alloc_t::allocate(n, pointer(0)); + ++allocations; + items_allocated += n; + return p; + } + + pointer allocate(const size_type n, const void * const) + { return allocate(n); } + + void deallocate(const pointer ptr, const size_type n) + { + ++frees; + items_freed += n; + base_alloc_t::deallocate(ptr, n); + } + + void set_limits(size_type max = 0) { + max_items = max; + } +}; + +template class Allocator = std::allocator> +class debug_allocator : public Allocator +{ +public: + typedef Allocator base_allocator_type; + typedef typename base_allocator_type::value_type value_type; + typedef typename base_allocator_type::pointer pointer; + typedef typename base_allocator_type::const_pointer const_pointer; + typedef typename base_allocator_type::reference reference; + typedef typename base_allocator_type::const_reference const_reference; + typedef typename base_allocator_type::size_type size_type; + typedef typename base_allocator_type::difference_type difference_type; + template struct rebind { + typedef debug_allocator other; + }; + + debug_allocator() throw() { } + debug_allocator(const debug_allocator &a) throw() : base_allocator_type( a ) { } + template + debug_allocator(const debug_allocator &a) throw() : base_allocator_type( Allocator( a ) ) { } + + pointer allocate(const size_type n, const void *hint = 0 ) { + pointer ptr = base_allocator_type::allocate( n, hint ); + std::memset( (void*)ptr, 0xE3E3E3E3, n * sizeof(value_type) ); + return ptr; + } +}; + +//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 +/** @ingroup memory_allocation */ +template class Allocator> +class debug_allocator : public Allocator { +public: + typedef Allocator base_allocator_type; + typedef typename base_allocator_type::value_type value_type; + typedef typename base_allocator_type::pointer pointer; + typedef typename base_allocator_type::const_pointer const_pointer; + template struct rebind { + typedef debug_allocator other; + }; +}; + +template class B1, typename T2, template class B2> +inline bool operator==( const debug_allocator &a, const debug_allocator &b) { + return static_cast< B1 >(a) == static_cast< B2 >(b); +} +template class B1, typename T2, template class B2> +inline bool operator!=( const debug_allocator &a, const debug_allocator &b) { + return static_cast< B1 >(a) != static_cast< B2 >(b); +} + +template class Allocator = std::allocator> +class stateful_allocator : public Allocator +{ + void* unique_pointer; + + template class Allocator1> + friend class stateful_allocator; +public: + typedef Allocator base_allocator_type; + typedef typename base_allocator_type::value_type value_type; + typedef typename base_allocator_type::pointer pointer; + typedef typename base_allocator_type::const_pointer const_pointer; + typedef typename base_allocator_type::reference reference; + typedef typename base_allocator_type::const_reference const_reference; + typedef typename base_allocator_type::size_type size_type; + typedef typename base_allocator_type::difference_type difference_type; + template struct rebind { + typedef stateful_allocator other; + }; + typedef pocma propagate_on_container_move_assignment; + + stateful_allocator() throw() : unique_pointer(this) { } + + template + stateful_allocator(const stateful_allocator &a) throw() : base_allocator_type( Allocator( a ) ), unique_pointer(a.uniqe_pointer) { } + + friend bool operator==(stateful_allocator const& lhs, stateful_allocator const& rhs){ + return lhs.unique_pointer == rhs.unique_pointer; + } + + friend bool operator!=(stateful_allocator const& rhs, stateful_allocator const& lhs){ + return !(lhs == rhs); + } + +}; + +template +class pmr_stateful_allocator +{ +private: + pmr_stateful_allocator& operator=(const pmr_stateful_allocator&); /* = deleted */ +public: + typedef T value_type; + typedef Harness::false_type propagate_on_container_move_assignment; + typedef Harness::false_type propagate_on_container_copy_assignment; + typedef Harness::false_type propagate_on_container_swap; + +// These types are required in C++03 +#if !__TBB_ALLOCATOR_TRAITS_PRESENT + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + template struct rebind { + typedef pmr_stateful_allocator other; + }; +#endif + + pmr_stateful_allocator() throw() : unique_pointer(this) {} + + pmr_stateful_allocator(const pmr_stateful_allocator &a) : unique_pointer(a.unique_pointer) {} + + template + pmr_stateful_allocator(const pmr_stateful_allocator &a) throw() : unique_pointer(a.unique_pointer) {} + + value_type* allocate( size_t n, const void* /*hint*/ = 0 ) { + return static_cast( malloc( n * sizeof(value_type) ) ); + } + + void deallocate( value_type* p, size_t ) { + free( p ); + } + +#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + //! Copy-construct value at location pointed to by p. + template + void construct(U *p, Args&&... args) + { + ::new((void *)p) U(std::forward(args)...); + } +#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC +#if __TBB_CPP11_RVALUE_REF_PRESENT + void construct(value_type* p, value_type&& value) { ::new((void*)(p)) value_type(std::move(value)); } +#endif + void construct(value_type* p, const value_type& value) { ::new((void*)(p)) value_type(value); } +#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + + //! Destroy value at location pointed to by p. + void destroy(value_type* p) { + p->~value_type(); + // suppress "unreferenced parameter" warnings by MSVC up to and including 2015 + tbb::internal::suppress_unused_warning(p); + } + + friend bool operator==(pmr_stateful_allocator const& lhs, pmr_stateful_allocator const& rhs){ + return lhs.unique_pointer == rhs.unique_pointer; + } + + friend bool operator!=(pmr_stateful_allocator const& rhs, pmr_stateful_allocator const& lhs){ + return !(lhs == rhs); + } + + void* unique_pointer; +}; + +// C++03 allocator doesn't have to be assignable or swappable, so +// tbb::internal::allocator_traits defines POCCA and POCS as false_type +#if __TBB_ALLOCATOR_TRAITS_PRESENT +#include "tbb/internal/_allocator_traits.h" // Need traits_true/false_type + +template +struct propagating_allocator : Allocator { + typedef POCMA propagate_on_container_move_assignment; + typedef POCCA propagate_on_container_copy_assignment; + typedef POCS propagate_on_container_swap; + bool* propagated_on_copy_assignment; + bool* propagated_on_move_assignment; + bool* propagated_on_swap; + bool* selected_on_copy_construction; + + template + struct rebind { + typedef propagating_allocator::type, + POCMA, POCCA, POCS> other; + }; + + propagating_allocator() : propagated_on_copy_assignment(NULL), + propagated_on_move_assignment(NULL), + propagated_on_swap(NULL), + selected_on_copy_construction(NULL) {} + + propagating_allocator(bool& poca, bool& poma, bool& pos, bool& soc) + : propagated_on_copy_assignment(&poca), + propagated_on_move_assignment(&poma), + propagated_on_swap(&pos), + selected_on_copy_construction(&soc) {} + + propagating_allocator(const propagating_allocator& other) + : Allocator(other), + propagated_on_copy_assignment(other.propagated_on_copy_assignment), + propagated_on_move_assignment(other.propagated_on_move_assignment), + propagated_on_swap(other.propagated_on_swap), + selected_on_copy_construction(other.selected_on_copy_construction) {} + + template + propagating_allocator(const propagating_allocator& other) + : Allocator(other), + propagated_on_copy_assignment(other.propagated_on_copy_assignment), + propagated_on_move_assignment(other.propagated_on_move_assignment), + propagated_on_swap(other.propagated_on_swap), + selected_on_copy_construction(other.selected_on_copy_construction) {} + + propagating_allocator& operator=(const propagating_allocator&) { + ASSERT(POCCA::value, "Allocator should not copy assign if pocca is false"); + if (propagated_on_copy_assignment) + *propagated_on_copy_assignment = true; + return *this; + } + +#if __TBB_CPP11_RVALUE_REF_PRESENT + propagating_allocator& operator=(propagating_allocator&&) { + ASSERT(POCMA::value, "Allocator should not move assign if pocma is false"); + if (propagated_on_move_assignment) + *propagated_on_move_assignment = true; + return *this; + } +#endif + + propagating_allocator select_on_container_copy_construction() const { + if (selected_on_copy_construction) + *selected_on_copy_construction = true; + return *this; + } +}; + +namespace propagating_allocators { +typedef tbb::tbb_allocator base_allocator; +typedef tbb::internal::traits_true_type true_type; +typedef tbb::internal::traits_false_type false_type; + +typedef propagating_allocator always_propagating_allocator; +typedef propagating_allocator never_propagating_allocator; +typedef propagating_allocator pocma_allocator; +typedef propagating_allocator pocca_allocator; +typedef propagating_allocator pocs_allocator; +} + +template +void swap(propagating_allocator& lhs, + propagating_allocator&) { + ASSERT(POCS::value, "Allocator should not swap if pocs is false"); + if (lhs.propagated_on_swap) + *lhs.propagated_on_swap = true; +} + +template +void test_allocator_traits_support() { + typedef typename ContainerType::allocator_type allocator_type; + typedef std::allocator_traits allocator_traits; + typedef typename allocator_traits::propagate_on_container_copy_assignment pocca_type; +#if __TBB_CPP11_RVALUE_REF_PRESENT + typedef typename allocator_traits::propagate_on_container_move_assignment pocma_type; +#endif + typedef typename allocator_traits::propagate_on_container_swap pocs_type; + + bool propagated_on_copy = false; + bool propagated_on_move = false; + bool propagated_on_swap = false; + bool selected_on_copy = false; + + allocator_type alloc(propagated_on_copy, propagated_on_move, propagated_on_swap, selected_on_copy); + + ContainerType c1(alloc), c2(c1); + ASSERT(selected_on_copy, "select_on_container_copy_construction function was not called"); + + c1 = c2; + ASSERT(propagated_on_copy == pocca_type::value, "Unexpected allocator propagation on copy assignment"); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + c2 = std::move(c1); + ASSERT(propagated_on_move == pocma_type::value, "Unexpected allocator propagation on move assignment"); +#endif + + c1.swap(c2); + ASSERT(propagated_on_swap == pocs_type::value, "Unexpected allocator propagation on swap"); +} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +class non_movable_object { + non_movable_object() {} +private: + non_movable_object(non_movable_object&&); + non_movable_object& operator=(non_movable_object&&); +}; + +template +void test_allocator_traits_with_non_movable_value_type() { + // Check, that if pocma is true, container allows move assignment without per-element move + typedef typename ContainerType::allocator_type allocator_type; + typedef std::allocator_traits allocator_traits; + typedef typename allocator_traits::propagate_on_container_move_assignment pocma_type; + ASSERT(pocma_type::value, "Allocator POCMA must be true for this test"); + allocator_type alloc; + ContainerType container1(alloc), container2(alloc); + container1 = std::move(container2); +} +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT + +template +class allocator_aware_data { +public: + static bool assert_on_constructions; + typedef Allocator allocator_type; + + allocator_aware_data(const allocator_type& allocator = allocator_type()) + : my_allocator(allocator), my_value(0) {} + allocator_aware_data(int v, const allocator_type& allocator = allocator_type()) + : my_allocator(allocator), my_value(v) {} + allocator_aware_data(const allocator_aware_data&) { + ASSERT(!assert_on_constructions, "Allocator should propagate to the data during copy construction"); + } + allocator_aware_data(allocator_aware_data&&) { + ASSERT(!assert_on_constructions, "Allocator should propagate to the data during move construction"); + } + allocator_aware_data(const allocator_aware_data& rhs, const allocator_type& allocator) + : my_allocator(allocator), my_value(rhs.my_value) {} + allocator_aware_data(allocator_aware_data&& rhs, const allocator_type& allocator) + : my_allocator(allocator), my_value(rhs.my_value) {} + + int value() const { return my_value; } +private: + allocator_type my_allocator; + int my_value; +}; + +template +bool allocator_aware_data::assert_on_constructions = false; + +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Workaround for overzealous compiler warnings + #pragma warning (pop) +#endif // warning 4267,4512,4355 is back + +namespace Harness { + + struct IsEqual { +#if __TBB_CPP11_SMART_POINTERS_PRESENT + template + static bool compare( const std::weak_ptr &t1, const std::weak_ptr &t2 ) { + // Compare real pointers. + return t1.lock().get() == t2.lock().get(); + } + template + static bool compare( const std::unique_ptr &t1, const std::unique_ptr &t2 ) { + // Compare real values. + return *t1 == *t2; + } + template + static bool compare( const std::pair< const std::weak_ptr, std::weak_ptr > &t1, + const std::pair< const std::weak_ptr, std::weak_ptr > &t2 ) { + // Compare real pointers. + return t1.first.lock().get() == t2.first.lock().get() && + t1.second.lock().get() == t2.second.lock().get(); + } +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ + template + static bool compare( const T1 &t1, const T2 &t2 ) { + return t1 == t2; + } + template + bool operator()( T1 &t1, T2 &t2) const { + return compare( (const T1&)t1, (const T2&)t2 ); + } + }; + +} // Harness +#endif // tbb_test_harness_allocator_H diff --git a/src/tbb/src/test/harness_allocator_overload.h b/src/tbb/src/test/harness_allocator_overload.h new file mode 100644 index 00000000..e94accdd --- /dev/null +++ b/src/tbb/src/test/harness_allocator_overload.h @@ -0,0 +1,35 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_test_harness_allocator_overload_H +#define tbb_test_harness_allocator_overload_H + +#include "../tbbmalloc/proxy.h" // for MALLOC_UNIXLIKE_OVERLOAD_ENABLED, MALLOC_ZONE_OVERLOAD_ENABLED +#include "tbb/tbb_config.h" // for __TBB_WIN8UI_SUPPORT + +// Skip configurations with unsupported system malloc overload: +// skip unsupported MSVCs, WIN8UI and MINGW (it doesn't define _MSC_VER), +// no support for MSVC 2015 and greater in debug for now, +// don't use defined(_MSC_VER), because result of using defined() in macro expansion is undefined +#define MALLOC_WINDOWS_OVERLOAD_ENABLED ((_WIN32||_WIN64) && !__TBB_WIN8UI_SUPPORT && _MSC_VER >= 1500 && !(_MSC_VER >= 1900 && _DEBUG)) + +// Skip configurations with unsupported system malloc overload: +// * overload via linking with -lmalloc_proxy is broken in offload, +// as the library is loaded too late in that mode, +// * LD_PRELOAD mechanism is broken in offload +#define HARNESS_SKIP_TEST ((!MALLOC_WINDOWS_OVERLOAD_ENABLED && !MALLOC_UNIXLIKE_OVERLOAD_ENABLED && !MALLOC_ZONE_OVERLOAD_ENABLED) || __TBB_MIC_OFFLOAD) + +#endif // tbb_test_harness_allocator_overload_H diff --git a/src/tbb/src/test/harness_assert.h b/src/tbb/src/test/harness_assert.h new file mode 100644 index 00000000..22a3fb95 --- /dev/null +++ b/src/tbb/src/test/harness_assert.h @@ -0,0 +1,37 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Just the assertion portion of the harness. +// This is useful for writing portions of tests that include +// the minimal number of necessary header files. +// +// The full "harness.h" must be included later. + +#ifndef harness_assert_H +#define harness_assert_H + +void ReportError( const char* filename, int line, const char* expression, const char* message); +void ReportWarning( const char* filename, int line, const char* expression, const char* message); + +#define ASSERT_CUSTOM(p,message,file,line) ((p)?(void)0:ReportError(file,line,#p,message)) +#define ASSERT(p,message) ASSERT_CUSTOM(p,message,__FILE__,__LINE__) +#define ASSERT_WARNING(p,message) ((p)?(void)0:ReportWarning(__FILE__,__LINE__,#p,message)) + +//! Compile-time error if x and y have different types +template +void AssertSameType( const T& /*x*/, const T& /*y*/ ) {} + +#endif /* harness_assert_H */ diff --git a/src/tbb/src/test/harness_bad_expr.h b/src/tbb/src/test/harness_bad_expr.h new file mode 100644 index 00000000..46f24d13 --- /dev/null +++ b/src/tbb/src/test/harness_bad_expr.h @@ -0,0 +1,73 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for checking __TBB_ASSERT checks inside TBB. +// This header is an optional part of the test harness. +// It assumes that "harness.h" has already been included. + +#define TRY_BAD_EXPR_ENABLED (TBB_USE_ASSERT && TBB_USE_EXCEPTIONS && !__TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN) + +#if TRY_BAD_EXPR_ENABLED + +//! Check that expression x raises assertion failure with message containing given substring. +/** Assumes that tbb::set_assertion_handler( AssertionFailureHandler ) was called earlier. */ +#define TRY_BAD_EXPR(x,substr) \ + { \ + const char* message = NULL; \ + bool okay = false; \ + try { \ + x; \ + } catch( AssertionFailure a ) { \ + okay = true; \ + message = a.message; \ + } \ + CheckAssertionFailure(__LINE__,#x,okay,message,substr); \ + } + +//! Exception object that holds a message. +struct AssertionFailure { + const char* message; + AssertionFailure( const char* filename, int line, const char* expression, const char* comment ); +}; + +AssertionFailure::AssertionFailure( const char* filename, int line, const char* expression, const char* comment ) : + message(comment) +{ + ASSERT(filename,"missing filename"); + ASSERT(0 + void operator()( const volatile T& location, U value ) const { + tbb::tick_count start = tbb::tick_count::now(); + double time_passed; + do { + time_passed = (tbb::tick_count::now()-start).seconds(); + if( time_passed < 0.0001 ) __TBB_Pause(10); else __TBB_Yield(); + } while( time_passed < my_limit && location == value); + my_limit -= time_passed; + } +}; +//! Spin WHILE the value of the variable is equal to a given value +/** T and U should be comparable types. */ +class WaitWhileEq { + //! Assignment not allowed + void operator=( const WaitWhileEq& ); +public: + template + void operator()( const volatile T& location, U value ) const { + tbb::internal::spin_wait_while_eq(location, value); + } +}; +class SpinBarrier +{ + unsigned numThreads; + tbb::atomic numThreadsFinished; // reached the barrier in this epoch + // the number of times the barrier was opened; TODO: move to a separate cache line + tbb::atomic epoch; + // a throwaway barrier can be used only once, then wait() becomes a no-op + bool throwaway; + + struct DummyCallback { + void operator() () const {} + template + void operator()( const T&, U) const {} + }; + + SpinBarrier( const SpinBarrier& ); // no copy ctor + void operator=( const SpinBarrier& ); // no assignment +public: + SpinBarrier( unsigned nthreads = 0, bool throwaway_ = false ) { + initialize(nthreads, throwaway_); + } + void initialize( unsigned nthreads, bool throwaway_ = false ) { + numThreads = nthreads; + numThreadsFinished = 0; + epoch = 0; + throwaway = throwaway_; + } + + // Returns whether this thread was the last to reach the barrier. + // onWaitCallback is called by a thread for waiting; + // onOpenBarrierCallback is called by the last thread before unblocking other threads. + template + bool custom_wait(const WaitEq &onWaitCallback, const Callback &onOpenBarrierCallback) + { + if (throwaway && epoch) + return false; + unsigned myEpoch = epoch; + unsigned myNumThreads = numThreads; // read it before the increment + int threadsLeft = myNumThreads - numThreadsFinished.fetch_and_increment() - 1; + ASSERT(threadsLeft>=0, "Broken barrier"); + if (threadsLeft > 0) { + /* this thread is not the last; wait until the epoch changes & return false */ + onWaitCallback(epoch, myEpoch); + return false; + } + /* This thread is the last one at the barrier in this epoch */ + onOpenBarrierCallback(); + /* reset the barrier, increment the epoch, and return true */ + threadsLeft = numThreadsFinished -= myNumThreads; + ASSERT( threadsLeft == 0, "Broken barrier"); + /* wakes up threads waiting to exit in this epoch */ + myEpoch -= epoch++; + ASSERT( myEpoch == 0, "Broken barrier"); + return true; + } + bool timed_wait_noerror(double n_seconds) { + custom_wait(TimedWaitWhileEq(n_seconds), DummyCallback()); + return n_seconds >= 0.0001; + } + bool timed_wait(double n_seconds, const char *msg="Time is out while waiting on a barrier") { + bool is_last = custom_wait(TimedWaitWhileEq(n_seconds), DummyCallback()); + ASSERT( n_seconds >= 0, msg); // TODO: refactor to avoid passing msg here and rising assertion + return is_last; + } + // onOpenBarrierCallback is called by the last thread before unblocking other threads. + template + bool wait(const Callback &onOpenBarrierCallback) { + return custom_wait(WaitWhileEq(), onOpenBarrierCallback); + } + bool wait(){ + return wait(DummyCallback()); + } + //! signal to the barrier, rather a semaphore functionality + bool signal_nowait() { + return custom_wait(DummyCallback(),DummyCallback()); + } +}; + +} + +#endif //harness_barrier_H diff --git a/src/tbb/src/test/harness_checktype.h b/src/tbb/src/test/harness_checktype.h new file mode 100644 index 00000000..89cf7d03 --- /dev/null +++ b/src/tbb/src/test/harness_checktype.h @@ -0,0 +1,95 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_tests_harness_checktype_H +#define tbb_tests_harness_checktype_H + +// type that checks construction and destruction. + +#ifndef __HARNESS_CHECKTYPE_DEFAULT_CTOR + #define __HARNESS_CHECKTYPE_DEFAULT_CTOR 1 +#endif + +template +class check_type : Harness::NoAfterlife { + Counter id; + bool am_ready; +public: + static tbb::atomic check_type_counter; + // if only non-default constructors are desired, set __HARNESS_CHECKTYPE_NODEFAULT_CTOR + check_type(Counter _n +#if __HARNESS_CHECKTYPE_DEFAULT_CTOR + = 0 +#endif + ) : id(_n), am_ready(false) { + ++check_type_counter; + } + + check_type(const check_type& other) : Harness::NoAfterlife(other) { + other.AssertLive(); + AssertLive(); + id = other.id; + am_ready = other.am_ready; + ++check_type_counter; + } + + operator int() const { return (int)my_id(); } + check_type& operator++() { ++id; return *this;; } + + ~check_type() { + AssertLive(); + --check_type_counter; + ASSERT(check_type_counter >= 0, "too many destructions"); + } + + check_type &operator=(const check_type &other) { + other.AssertLive(); + AssertLive(); + id = other.id; + am_ready = other.am_ready; + return *this; + } + + Counter my_id() const { AssertLive(); return id; } + bool is_ready() { AssertLive(); return am_ready; } + void function() { + AssertLive(); + if( id == (Counter)0 ) { + id = (Counter)1; + am_ready = true; + } + } + +}; + +template +tbb::atomic check_type::check_type_counter; + +// provide a class that for a check_type will initialize the counter on creation, and on +// destruction will check that the constructions and destructions of check_type match. +template +struct Check { + Check() {} // creation does nothing + ~Check() {} // destruction checks nothing +}; + +template +struct Check > { + Check() { check_type::check_type_counter = 0; } + ~Check() { ASSERT(check_type::check_type_counter == 0, "check_type constructions and destructions don't match"); } +}; + +#endif // tbb_tests_harness_checktype_H diff --git a/src/tbb/src/test/harness_concurrency.h b/src/tbb/src/test/harness_concurrency.h new file mode 100644 index 00000000..07f147b1 --- /dev/null +++ b/src/tbb/src/test/harness_concurrency.h @@ -0,0 +1,101 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_tests_harness_concurrency_H +#define tbb_tests_harness_concurrency_H + +#if _WIN32||_WIN64 +#include "tbb/machine/windows_api.h" +#elif __linux__ +#include +#include +#include +#include +#elif __FreeBSD__ +#include +#include +#include +#include // Required by +#include +#endif + +#include + +namespace Harness { + static int maxProcs = 0; + static int GetMaxProcs() { + if ( !maxProcs ) { +#if _WIN32||_WIN64 + SYSTEM_INFO si; + GetNativeSystemInfo(&si); + maxProcs = si.dwNumberOfProcessors; +#elif __linux__ + maxProcs = get_nprocs(); +#else /* __FreeBSD__ */ + maxProcs = sysconf(_SC_NPROCESSORS_ONLN); +#endif + } + return maxProcs; + } + + int LimitNumberOfThreads(int max_threads) { + ASSERT( max_threads >= 1 , "The limited number of threads should be positive." ); + maxProcs = GetMaxProcs(); + if ( maxProcs < max_threads ) + // Suppose that process mask is not set so the number of available threads equals maxProcs + return maxProcs; + +#if _WIN32||_WIN64 + ASSERT( max_threads <= 64 , "LimitNumberOfThreads doesn't support max_threads to be more than 64 on Windows." ); + DWORD_PTR mask = 1; + for ( int i = 1; i < max_threads; ++i ) + mask |= mask << 1; + bool err = !SetProcessAffinityMask( GetCurrentProcess(), mask ); +#else /* !WIN */ +#if __linux__ + typedef cpu_set_t mask_t; +#if __TBB_MAIN_THREAD_AFFINITY_BROKEN +#define setaffinity(mask) sched_setaffinity(0 /*get the mask of the calling thread*/, sizeof(mask_t), &mask) +#else +#define setaffinity(mask) sched_setaffinity(getpid(), sizeof(mask_t), &mask) +#endif +#else /* __FreeBSD__ */ + typedef cpuset_t mask_t; +#if __TBB_MAIN_THREAD_AFFINITY_BROKEN +#define setaffinity(mask) cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask_t), &mask) +#else +#define setaffinity(mask) cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(mask_t), &mask) +#endif +#endif /* __FreeBSD__ */ + mask_t newMask; + CPU_ZERO(&newMask); + + int maskSize = (int)sizeof(mask_t) * CHAR_BIT; + ASSERT_WARNING( maskSize >= maxProcs, "The mask size doesn't seem to be big enough to call setaffinity. The call may return an error." ); + + ASSERT( max_threads <= (int)sizeof(mask_t) * CHAR_BIT , "The mask size is not enough to set the requested number of threads." ); + for ( int i = 0; i < max_threads; ++i ) + CPU_SET( i, &newMask ); + int err = setaffinity( newMask ); +#endif /* !WIN */ + ASSERT( !err, "Setting process affinity failed" ); + + return max_threads; + } + +} // namespace Harness + +#endif /* tbb_tests_harness_concurrency_H */ diff --git a/src/tbb/src/test/harness_concurrency_tracker.h b/src/tbb/src/test/harness_concurrency_tracker.h new file mode 100644 index 00000000..f6d56f28 --- /dev/null +++ b/src/tbb/src/test/harness_concurrency_tracker.h @@ -0,0 +1,170 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_tests_harness_concurrency_tracker_H +#define tbb_tests_harness_concurrency_tracker_H + +#include "harness_assert.h" +#include "harness_barrier.h" +#include "tbb/atomic.h" +#include "../tbb/tls.h" +// Note: This file is used by RML tests which do not link TBB. +// Functionality that requires TBB binaries must be guarded by !__TBB_NO_IMPLICIT_LINKAGE +#if !defined(__TBB_NO_IMPLICIT_LINKAGE) +#include "tbb/mutex.h" +#include "tbb/task.h" +#include "tbb/combinable.h" +#include "tbb/parallel_for.h" +#include // for std::plus +#include "harness.h" // for Harness::NoCopy +#endif + +namespace Harness { + +static tbb::atomic ctInstantParallelism; +static tbb::atomic ctPeakParallelism; +static tbb::internal::tls ctNested; + +class ConcurrencyTracker { + bool m_Outer; + + static void Started () { + unsigned p = ++ctInstantParallelism; + unsigned q = ctPeakParallelism; + while( q

0, "Mismatched call to ConcurrencyTracker::Stopped()" ); + --ctInstantParallelism; + } +public: + ConcurrencyTracker() : m_Outer(false) { + uintptr_t nested = ctNested; + ASSERT (nested == 0 || nested == 1, NULL); + if ( !ctNested ) { + Started(); + m_Outer = true; + ctNested = 1; + } + } + ~ConcurrencyTracker() { + if ( m_Outer ) { + Stopped(); + ctNested = 0; + } + } + + static unsigned PeakParallelism() { return ctPeakParallelism; } + static unsigned InstantParallelism() { return ctInstantParallelism; } + + static void Reset() { + ASSERT (ctInstantParallelism == 0, "Reset cannot be called when concurrency tracking is underway"); + ctInstantParallelism = ctPeakParallelism = 0; + } +}; // ConcurrencyTracker + +#if !defined(__TBB_NO_IMPLICIT_LINKAGE) +struct ExactConcurrencyLevel : NoCopy { + typedef tbb::combinable Combinable; +private: + Harness::SpinBarrier *myBarrier; + // count unique worker threads + Combinable *myUniqueThreads; + mutable tbb::atomic myActiveBodyCnt; + // output parameter for parallel_for body to report that max is reached + mutable bool myReachedMax; + // zero timeout means no barrier is used during concurrency level detection + const double myTimeout; + const size_t myConcLevel; + const bool myCrashOnFail; + + static tbb::mutex global_mutex; + + ExactConcurrencyLevel(double timeout, size_t concLevel, Combinable *uniq, bool crashOnFail) : + myBarrier(NULL), myUniqueThreads(uniq), myReachedMax(false), + myTimeout(timeout), myConcLevel(concLevel), myCrashOnFail(crashOnFail) { + myActiveBodyCnt = 0; + } + bool run() { + const int LOOP_ITERS = 100; + tbb::combinable uniq; + Harness::SpinBarrier barrier((unsigned)myConcLevel, /*throwaway=*/true); + if (myTimeout != 0.) + myBarrier = &barrier; + if (!myUniqueThreads) + myUniqueThreads = &uniq; + tbb::parallel_for((size_t)0, myConcLevel*LOOP_ITERS, *this, tbb::simple_partitioner()); + return myReachedMax; + } +public: + void operator()(size_t) const { + size_t v = ++myActiveBodyCnt; + ASSERT(v <= myConcLevel, "Number of active bodies is too high."); + if (v == myConcLevel) // record that the max expected concurrency was observed + myReachedMax = true; + // try to get barrier when 1st time in the thread + if (myBarrier && !myBarrier->timed_wait_noerror(myTimeout)) + ASSERT(!myCrashOnFail, "Timeout was detected."); + + myUniqueThreads->local() = 1; + for (int i=0; i<100; i++) + __TBB_Pause(1); + --myActiveBodyCnt; + } + + enum Mode { + None, + // When multiple blocking checks are performed, there might be not enough + // concurrency for all of them. Serialize check() calls. + Serialize + }; + + // check that we have never got more than concLevel threads, + // and that in some moment we saw exactly concLevel threads + static void check(size_t concLevel, Mode m = None) { + ExactConcurrencyLevel o(30., concLevel, NULL, /*crashOnFail=*/true); + + tbb::mutex::scoped_lock lock; + if (m == Serialize) + lock.acquire(global_mutex); + bool ok = o.run(); + ASSERT(ok, NULL); + } + + static bool isEqual(size_t concLevel) { + ExactConcurrencyLevel o(3., concLevel, NULL, /*crashOnFail=*/false); + return o.run(); + } + + static void checkLessOrEqual(size_t concLevel, tbb::combinable *unique) { + ExactConcurrencyLevel o(0., concLevel, unique, /*crashOnFail=*/true); + + o.run(); // ignore result, as without a barrier it is not reliable + const size_t num = unique->combine(std::plus()); + ASSERT(num<=concLevel, "Too many workers observed."); + } +}; + +tbb::mutex ExactConcurrencyLevel::global_mutex; + +#endif /* !defined(__TBB_NO_IMPLICIT_LINKAGE) */ + +} // namespace Harness + +#endif /* tbb_tests_harness_concurrency_tracker_H */ diff --git a/src/tbb/src/test/harness_cpu.h b/src/tbb/src/test/harness_cpu.h new file mode 100644 index 00000000..092cc03e --- /dev/null +++ b/src/tbb/src/test/harness_cpu.h @@ -0,0 +1,116 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for simple estimate of CPU time being used by a program. +// This header is an optional part of the test harness. +// It assumes that "harness_assert.h" has already been included. + +#if _WIN32 + #include +#else + #include + #include +#endif + +//! Return time (in seconds) spent by the current process in user mode. +/* Returns 0 if not implemented on platform. */ +static double GetCPUUserTime() { +#if __TBB_WIN8UI_SUPPORT + return 0; +#elif _WIN32 + FILETIME my_times[4]; + bool status = GetProcessTimes(GetCurrentProcess(), my_times, my_times+1, my_times+2, my_times+3)!=0; + ASSERT( status, NULL ); + LARGE_INTEGER usrtime; + usrtime.LowPart = my_times[3].dwLowDateTime; + usrtime.HighPart = my_times[3].dwHighDateTime; + return double(usrtime.QuadPart)*1E-7; +#else + // Generic UNIX, including __APPLE__ + + // On Linux, there is no good way to get CPU usage info for the current process: + // getrusage(RUSAGE_SELF, ...) that is used now only returns info for the calling thread; + // getrusage(RUSAGE_CHILDREN, ...) only counts for finished children threads; + // tms_utime and tms_cutime got with times(struct tms*) are equivalent to the above items; + // finally, /proc/self/task//stat doesn't exist on older kernels + // and it isn't quite convenient to read it for every task_id. + + struct rusage resources; + bool status = getrusage(RUSAGE_SELF, &resources)==0; + ASSERT( status, NULL ); + return (double(resources.ru_utime.tv_sec)*1E6 + double(resources.ru_utime.tv_usec))*1E-6; +#endif +} + +#include "tbb/tick_count.h" +#include + +// The resolution of GetCPUUserTime is 10-15 ms or so; waittime should be a few times bigger. +const double WAITTIME = 0.1; // in seconds, i.e. 100 ms +const double THRESHOLD = WAITTIME/100; + +static void TestCPUUserTime( int nthreads, int nactive = 1 ) { + // The test will always pass on Linux; read the comments in GetCPUUserTime for details + // Also it will not detect spinning issues on systems with only one processing core. + + int nworkers = nthreads-nactive; + if( !nworkers ) return; + double lastusrtime = GetCPUUserTime(); + if( !lastusrtime ) return; + + static double minimal_waittime = WAITTIME, + maximal_waittime = WAITTIME * 10; + double usrtime_delta; + double waittime_delta; + tbb::tick_count stamp = tbb::tick_count::now(); + volatile intptr_t k = (intptr_t)&usrtime_delta; + // wait for GetCPUUserTime update + while( (usrtime_delta=GetCPUUserTime()-lastusrtime) < THRESHOLD ) { + for ( int i = 0; i < 1000; ++i ) ++k; // do fake work without which user time can stall + if ( (waittime_delta = (tbb::tick_count::now()-stamp).seconds()) > maximal_waittime ) { + REPORT( "Warning: %.2f sec elapsed but user mode time is still below its threshold (%g < %g)\n", + waittime_delta, usrtime_delta, THRESHOLD ); + break; + } + } + lastusrtime += usrtime_delta; + + // Wait for workers to go sleep + stamp = tbb::tick_count::now(); + while( ((waittime_delta=(tbb::tick_count::now()-stamp).seconds()) < minimal_waittime) + || ((usrtime_delta=GetCPUUserTime()-lastusrtime) < THRESHOLD) ) + { + for ( int i = 0; i < 1000; ++i ) ++k; // do fake work without which user time can stall + if ( waittime_delta > maximal_waittime ) { + REPORT( "Warning: %.2f sec elapsed but GetCPUUserTime reported only %g sec\n", waittime_delta, usrtime_delta ); + break; + } + } + + // Test that all workers sleep when no work. + while( nactive>1 && usrtime_delta-nactive*waittime_delta<0 ) { + // probably the number of active threads was mispredicted + --nactive; ++nworkers; + } + double avg_worker_usrtime = (usrtime_delta-nactive*waittime_delta)/nworkers; + + if( avg_worker_usrtime > waittime_delta/2 ) + REPORT( "ERROR: %d worker threads are spinning; waittime: %g; usrtime: %g; avg worker usrtime: %g\n", + nworkers, waittime_delta, usrtime_delta, avg_worker_usrtime); + else + REMARK("%d worker threads; waittime: %g; usrtime: %g; avg worker usrtime: %g\n", + nworkers, waittime_delta, usrtime_delta, avg_worker_usrtime); +} diff --git a/src/tbb/src/test/harness_defs.h b/src/tbb/src/test/harness_defs.h new file mode 100644 index 00000000..f7363dad --- /dev/null +++ b/src/tbb/src/test/harness_defs.h @@ -0,0 +1,220 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_harness_defs_H +#define __TBB_harness_defs_H + +#include "tbb/tbb_config.h" +#if __FreeBSD__ +#include // for __FreeBSD_version +#endif + +#if __TBB_TEST_PIC && !__PIC__ +#define __TBB_TEST_SKIP_PIC_MODE 1 +#else +#define __TBB_TEST_SKIP_PIC_MODE 0 +#endif + +// no need to test GCC builtins mode on ICC +#define __TBB_TEST_SKIP_GCC_BUILTINS_MODE ( __TBB_TEST_BUILTINS && (!__TBB_GCC_BUILTIN_ATOMICS_PRESENT || __INTEL_COMPILER) ) + +#define __TBB_TEST_SKIP_ICC_BUILTINS_MODE ( __TBB_TEST_BUILTINS && !__TBB_ICC_BUILTIN_ATOMICS_PRESENT ) + +#ifndef TBB_USE_GCC_BUILTINS + // Force TBB to use GCC intrinsics port, but not on ICC, as no need + #define TBB_USE_GCC_BUILTINS ( __TBB_TEST_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT && !__INTEL_COMPILER ) +#endif + +#ifndef TBB_USE_ICC_BUILTINS + // Force TBB to use ICC c++11 style intrinsics port + #define TBB_USE_ICC_BUILTINS ( __TBB_TEST_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT ) +#endif + +#if (_WIN32 && !__TBB_WIN8UI_SUPPORT) || (__linux__ && !__ANDROID__ && !__bg__) || __FreeBSD_version >= 701000 +#define __TBB_TEST_SKIP_AFFINITY 0 +#else +#define __TBB_TEST_SKIP_AFFINITY 1 +#endif + +#if __INTEL_COMPILER + #define __TBB_CPP11_REFERENCE_WRAPPER_PRESENT ( __INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200 && \ + ( _MSC_VER >= 1600 || __TBB_GLIBCXX_VERSION >= 40400 || ( _LIBCPP_VERSION && __cplusplus >= 201103L ) ) ) + #define __TBB_RANGE_BASED_FOR_PRESENT ( __INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1300 ) + #define __TBB_SCOPED_ENUM_PRESENT ( __INTEL_CXX11_MODE__ && __INTEL_COMPILER > 1100 ) +#elif __clang__ + #define __TBB_CPP11_REFERENCE_WRAPPER_PRESENT ( __cplusplus >= 201103L && (__TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION) ) + #define __TBB_RANGE_BASED_FOR_PRESENT ( __has_feature(__cxx_range_for) ) + #define __TBB_SCOPED_ENUM_PRESENT ( __has_feature(cxx_strong_enums) ) +#elif __GNUC__ + #define __TBB_CPP11_REFERENCE_WRAPPER_PRESENT ( __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400 ) + #define __TBB_RANGE_BASED_FOR_PRESENT ( __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500 ) + #define __TBB_SCOPED_ENUM_PRESENT ( __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400 ) + #define __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT (__TBB_GCC_VERSION >= 60100) +#elif _MSC_VER + #define __TBB_CPP11_REFERENCE_WRAPPER_PRESENT ( _MSC_VER >= 1600 ) + #define __TBB_RANGE_BASED_FOR_PRESENT ( _MSC_VER >= 1700 ) + #define __TBB_SCOPED_ENUM_PRESENT ( _MSC_VER >= 1700 ) +#endif + +#define __TBB_CPP14_GENERIC_LAMBDAS_PRESENT (__cpp_generic_lambdas >= 201304 ) + +#define __TBB_TEST_SKIP_LAMBDA (__TBB_ICC_13_0_CPP11_STDLIB_SUPPORT_BROKEN || !__TBB_CPP11_LAMBDAS_PRESENT) + +#if __GNUC__ && __ANDROID__ +// On Android* OS, GCC does not support _thread keyword + #define __TBB_THREAD_LOCAL_VARIABLES_PRESENT 0 +#else + #define __TBB_THREAD_LOCAL_VARIABLES_PRESENT 1 +#endif + +// ICC has a bug in assumptions of the modifications made via atomic pointer +#define __TBB_ICC_BUILTIN_ATOMICS_POINTER_ALIASING_BROKEN (TBB_USE_ICC_BUILTINS && __INTEL_COMPILER < 1400 && __INTEL_COMPILER > 1200) + +// clang on Android/IA-32 fails on exception thrown from static move constructor +#define __TBB_CPP11_EXCEPTION_IN_STATIC_TEST_BROKEN (__ANDROID__ && __SIZEOF_POINTER__==4 && __clang__) + +// MSVC 2013 is unable to properly resolve call to overloaded operator= with std::initializer_list argument for std::pair list elements +// clang on Android/IA-32 fails on "std::vector> vd{{1,1},{1,1},{1,1}};" line in release mode +#define __TBB_CPP11_INIT_LIST_TEST_BROKEN (_MSC_VER <= 1800 && _MSC_VER && !__INTEL_COMPILER) || (__ANDROID__ && __TBB_x86_32 && __clang__) +// MSVC 2013 is unable to manage lifetime of temporary objects passed to a std::initializer_list constructor properly +#define __TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN (_MSC_FULL_VER < 180030501 && _MSC_VER && !__INTEL_COMPILER) + +// Implementation of C++11 std::placeholders in libstdc++ coming with GCC prior to 4.5 reveals bug in Intel(R) C++ Compiler 13 causing "multiple definition" link errors. +#define __TBB_CPP11_STD_PLACEHOLDERS_LINKAGE_BROKEN ((__INTEL_COMPILER == 1300 || __INTEL_COMPILER == 1310) && __GXX_EXPERIMENTAL_CXX0X__ && __GLIBCXX__ && __TBB_GLIBCXX_VERSION < 40500) + +// Intel C++ Compiler has an issue when a scoped enum with a specified underlying type has negative values. +#define __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_NEGATIVE_VALUE_BROKEN ( _MSC_VER && !__TBB_DEBUG && __INTEL_COMPILER && __INTEL_COMPILER <= 1500 ) +// Intel C++ Compiler has an issue with __atomic_load_explicit from a scoped enum with a specified underlying type. +#define __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_ATOMIC_LOAD_BROKEN ( TBB_USE_ICC_BUILTINS && !__TBB_DEBUG && __INTEL_COMPILER && __INTEL_COMPILER <= 1500 ) + +// Unable to use constexpr member functions to initialize compile time constants +#define __TBB_CONSTEXPR_MEMBER_FUNCTION_BROKEN (__INTEL_COMPILER == 1500) +// Some versions of MSVC do not do compile-time initialization of static variables with constexpr constructors in debug mode +#define __TBB_STATIC_CONSTEXPR_INIT_BROKEN (_MSC_VER >= 1900 && _MSC_VER <= 1914 && !__INTEL_COMPILER && _DEBUG) + +#if __GNUC__ && __ANDROID__ + #define __TBB_EXCEPTION_TYPE_INFO_BROKEN ( __TBB_GCC_VERSION < 40600 ) +#elif _MSC_VER + #define __TBB_EXCEPTION_TYPE_INFO_BROKEN ( _MSC_VER < 1400 ) +#else + #define __TBB_EXCEPTION_TYPE_INFO_BROKEN 0 +#endif + +// a function ptr cannot be converted to const T& template argument without explicit cast +#define __TBB_FUNC_PTR_AS_TEMPL_PARAM_BROKEN ( ((__linux__ || __APPLE__) && __INTEL_COMPILER && __INTEL_COMPILER < 1100) || __SUNPRO_CC ) + +#define __TBB_UNQUALIFIED_CALL_OF_DTOR_BROKEN (__GNUC__==3 && __GNUC_MINOR__<=3) + +#define __TBB_CAS_8_CODEGEN_BROKEN (__TBB_x86_32 && __PIC__ && __TBB_GCC_VERSION == 40102 && !__INTEL_COMPILER) + +#define __TBB_THROW_FROM_DTOR_BROKEN (__clang__ && __apple_build_version__ && __apple_build_version__ < 5000279) + +// std::uncaught_exception is broken on some version of stdlibc++ (it returns true with no active exception) +#define __TBB_STD_UNCAUGHT_EXCEPTION_BROKEN (__TBB_GLIBCXX_VERSION == 40407) + +#if __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN + #define _EXCEPTION_PTR_H /* prevents exception_ptr.h inclusion */ + #define _GLIBCXX_NESTED_EXCEPTION_H /* prevents nested_exception.h inclusion */ +#endif + +// TODO: Investigate the cases that require this macro. +#define __TBB_COMPLICATED_ADL_BROKEN ( __GNUC__ && __TBB_GCC_VERSION < 40400 ) + +// Intel C++ Compiler fails to compile the comparison of tuples in some cases +#if __INTEL_COMPILER && __INTEL_COMPILER < 1700 + #define __TBB_TUPLE_COMPARISON_COMPILATION_BROKEN (__TBB_GLIBCXX_VERSION >= 40800 || __MIC__) +#endif + +// Intel C++ Compiler fails to compile std::reference in some cases +#if __INTEL_COMPILER && __INTEL_COMPILER < 1600 || __INTEL_COMPILER == 1600 && __INTEL_COMPILER_UPDATE <= 1 + #define __TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN (__TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION <= 50101 || __MIC__) +#endif + +// Intel C++ Compiler fails to generate non-throwing move members for a class inherited from template +#define __TBB_NOTHROW_MOVE_MEMBERS_IMPLICIT_GENERATION_BROKEN \ + (__INTEL_COMPILER>=1600 && __INTEL_COMPILER<=1900 || __INTEL_COMPILER==1500 && __INTEL_COMPILER_UPDATE>3) + +// std::is_copy_constructible::value returns 'true' for non copyable type when MSVC compiler is used. +#define __TBB_IS_COPY_CONSTRUCTIBLE_BROKEN ( _MSC_VER && (_MSC_VER <= 1700 || _MSC_VER <= 1800 && !__INTEL_COMPILER) ) + +// GCC 4.7 and 4.8 might fail to take an address of overloaded template function (bug 57043) +#if __GNUC__ && !__INTEL_COMPILER && !__clang__ + #define __TBB_GCC_OVERLOADED_TEMPLATE_FUNCTION_ADDRESS_BROKEN \ + (__TBB_GCC_VERSION>=40700 && __TBB_GCC_VERSION<40704 || __TBB_GCC_VERSION>=40800 && __TBB_GCC_VERSION<40803 ) +#endif + +// Swapping of scoped_allocator_adaptors is broken on GCC 4.9 and lower and on Android for Windows +// Allocator propagation into std::pair is broken for Apple clang, lower then 9.0 +// Compilation of header is broken for Visual Studio 2017 with ICC 17.8 +#define __TBB_SCOPED_ALLOCATOR_BROKEN (__TBB_GCC_VERSION <= 50100 || (__APPLE__ && __TBB_CLANG_VERSION < 90000) || \ + (__FreeBSD__ && __TBB_CLANG_VERSION <= 60000) || \ + (__ANDROID__ && (_WIN32 || _WIN64)) || \ + (_MSC_VER && _MSC_VER == 1912 && __INTEL_COMPILER == 1700)) + + + +// The tuple-based tests with more inputs take a long time to compile. If changes +// are made to the tuple implementation or any switch that controls it, or if testing +// with a new platform implementation of std::tuple, the test should be compiled with +// MAX_TUPLE_TEST_SIZE >= 10 (or the largest number of elements supported) to ensure +// all tuple sizes are tested. Expect a very long compile time. +#ifndef MAX_TUPLE_TEST_SIZE + #if TBB_USE_DEBUG + #define MAX_TUPLE_TEST_SIZE 3 + #else + #define MAX_TUPLE_TEST_SIZE 5 + #endif +#else + #if _MSC_VER +// test sizes <= 8 don't get "decorated name length exceeded" errors. (disable : 4503) + #if MAX_TUPLE_TEST_SIZE > 8 + #undef MAX_TUPLE_TEST_SIZE + #define MAX_TUPLE_TEST_SIZE 8 + #endif + #endif + #if MAX_TUPLE_TEST_SIZE > __TBB_VARIADIC_MAX + #undef MAX_TUPLE_TEST_SIZE + #define MAX_TUPLE_TEST_SIZE __TBB_VARIADIC_MAX + #endif +#endif + +#if __TBB_CPF_BUILD + #ifndef TBB_PREVIEW_FLOW_GRAPH_FEATURES + #define TBB_PREVIEW_FLOW_GRAPH_FEATURES 1 + #endif + #ifndef TBB_PREVIEW_FLOW_GRAPH_TRACE + #define TBB_PREVIEW_FLOW_GRAPH_TRACE 1 + #endif + #ifndef TBB_PREVIEW_ALGORITHM_TRACE + #define TBB_PREVIEW_ALGORITHM_TRACE 1 + #endif + #ifndef TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + #define TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR 1 + #endif +#endif + +namespace Harness { + //! Utility template function to prevent "unused" warnings by various compilers. + template void suppress_unused_warning( const T& ) {} + + //TODO: unify with one in tbb::internal + //! Utility helper structure to ease overload resolution + template struct int_to_type {}; +} + +const unsigned MByte = 1024*1024; + +#endif /* __TBB_harness_defs_H */ diff --git a/src/tbb/src/test/harness_dynamic_libs.h b/src/tbb/src/test/harness_dynamic_libs.h new file mode 100644 index 00000000..e4a1055c --- /dev/null +++ b/src/tbb/src/test/harness_dynamic_libs.h @@ -0,0 +1,124 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/tbb_config.h" + +// Include this header file before harness.h for HARNESS_SKIP_TEST to take effect +#if !__TBB_DYNAMIC_LOAD_ENABLED +#define HARNESS_SKIP_TEST 1 +#else + +#if _WIN32 || _WIN64 +#include "tbb/machine/windows_api.h" +#else +#include +#endif +#include "harness_assert.h" + +namespace Harness { + +#if TBB_USE_DEBUG +#define SUFFIX1 "_debug" +#define SUFFIX2 +#else +#define SUFFIX1 +#define SUFFIX2 "_debug" +#endif /* TBB_USE_DEBUG */ + +#if _WIN32||_WIN64 +#define PREFIX +#define EXT ".dll" +#else +#define PREFIX "lib" +#if __APPLE__ +#define EXT ".dylib" +// Android SDK build system does not support .so file name versioning +#elif __FreeBSD__ || __NetBSD__ || __sun || _AIX || __ANDROID__ +#define EXT ".so" +#elif __linux__ // Order of these elif's matters! +#define EXT __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) +#else +#error Unknown OS +#endif +#endif + +// Form the names of the TBB memory allocator binaries. +#define MALLOCLIB_NAME1 PREFIX "tbbmalloc" SUFFIX1 EXT +#define MALLOCLIB_NAME2 PREFIX "tbbmalloc" SUFFIX2 EXT + +#if _WIN32 || _WIN64 +typedef HMODULE LIBRARY_HANDLE; +#else +typedef void *LIBRARY_HANDLE; +#endif + +#if _WIN32 || _WIN64 +#define TEST_LIBRARY_NAME(base) base".dll" +#elif __APPLE__ +#define TEST_LIBRARY_NAME(base) base".dylib" +#else +#define TEST_LIBRARY_NAME(base) base".so" +#endif + +LIBRARY_HANDLE OpenLibrary(const char *name) +{ +#if _WIN32 || _WIN64 +#if __TBB_WIN8UI_SUPPORT + TCHAR wlibrary[MAX_PATH]; + if ( MultiByteToWideChar(CP_UTF8, 0, name, -1, wlibrary, MAX_PATH) == 0 ) return false; + return :: LoadPackagedLibrary( wlibrary, 0 ); +#else + return ::LoadLibrary(name); +#endif +#else + return dlopen(name, RTLD_NOW|RTLD_GLOBAL); +#endif +} + +void CloseLibrary(LIBRARY_HANDLE lib) +{ +#if _WIN32 || _WIN64 + BOOL ret = FreeLibrary(lib); + ASSERT(ret, "FreeLibrary must be successful"); +#else + int ret = dlclose(lib); + ASSERT(ret == 0, "dlclose must be successful"); +#endif +} + +typedef void (*FunctionAddress)(); + +template +void GetAddress(Harness::LIBRARY_HANDLE lib, const char *name, FunctionPointer& func) +{ +#if _WIN32 || _WIN64 + func = (FunctionPointer)(void*)GetProcAddress(lib, name); +#else + func = (FunctionPointer)dlsym(lib, name); +#endif + ASSERT(func, "Can't find required symbol in dynamic library"); +} + +FunctionAddress GetAddress(Harness::LIBRARY_HANDLE lib, const char *name) +{ + FunctionAddress func; + GetAddress(lib, name, func); + return func; +} + +} // namespace Harness + +#endif // __TBB_DYNAMIC_LOAD_ENABLED diff --git a/src/tbb/src/test/harness_eh.h b/src/tbb/src/test/harness_eh.h new file mode 100644 index 00000000..e05d63f2 --- /dev/null +++ b/src/tbb/src/test/harness_eh.h @@ -0,0 +1,313 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include "tbb/tbb_exception.h" +#include "tbb/atomic.h" +#if USE_TASK_SCHEDULER_OBSERVER +#include "tbb/task_scheduler_observer.h" +#endif +#include "harness.h" +#include "harness_concurrency_tracker.h" + +int g_NumThreads = 0; +Harness::tid_t g_Master = 0; +const char * g_Orig_Wakeup_Msg = "Missed wakeup or machine is overloaded?"; +const char * g_Wakeup_Msg = g_Orig_Wakeup_Msg; + +tbb::atomic g_CurExecuted, + g_ExecutedAtLastCatch, + g_ExecutedAtFirstCatch, + g_ExceptionsThrown, + g_MasterExecutedThrow, // number of times master entered exception code + g_NonMasterExecutedThrow, // number of times nonmaster entered exception code + g_PipelinesStarted; +volatile bool g_ExceptionCaught = false, + g_UnknownException = false; + +#if USE_TASK_SCHEDULER_OBSERVER +tbb::atomic g_ActualMaxThreads; +tbb::atomic g_ActualCurrentThreads; +#endif + +volatile bool g_ThrowException = true, + // g_Flog is true for nested construct tests with catches (exceptions are not allowed to + // propagate to the tbb construct itself.) + g_Flog = false, + g_MasterExecuted = false, + g_NonMasterExecuted = false; + +bool g_ExceptionInMaster = false; +bool g_SolitaryException = false; +bool g_NestedPipelines = false; + +//! Number of exceptions propagated into the user code (i.e. intercepted by the tests) +tbb::atomic g_NumExceptionsCaught; + +//----------------------------------------------------------- + +#if USE_TASK_SCHEDULER_OBSERVER +class eh_test_observer : public tbb::task_scheduler_observer { +public: + void on_scheduler_entry(bool is_worker) __TBB_override { + if(is_worker) { // we've already counted the master + size_t p = ++g_ActualCurrentThreads; + size_t q = g_ActualMaxThreads; + while(q < p) { + q = g_ActualMaxThreads.compare_and_swap(p,q); + } + } + else { + // size_t q = g_ActualMaxThreads; + } + } + void on_scheduler_exit(bool is_worker) __TBB_override { + if(is_worker) { + --g_ActualCurrentThreads; + } + } +}; +#endif +//----------------------------------------------------------- + +inline void ResetEhGlobals ( bool throwException = true, bool flog = false ) { + Harness::ConcurrencyTracker::Reset(); + g_CurExecuted = g_ExecutedAtLastCatch = g_ExecutedAtFirstCatch = 0; + g_ExceptionCaught = false; + g_UnknownException = false; + g_NestedPipelines = false; + g_ThrowException = throwException; + g_MasterExecutedThrow = 0; + g_NonMasterExecutedThrow = 0; + g_Flog = flog; + g_MasterExecuted = false; + g_NonMasterExecuted = false; +#if USE_TASK_SCHEDULER_OBSERVER + g_ActualMaxThreads = 1; // count master + g_ActualCurrentThreads = 1; // count master +#endif + g_ExceptionsThrown = g_NumExceptionsCaught = g_PipelinesStarted = 0; +} + +#if TBB_USE_EXCEPTIONS +class test_exception : public std::exception { + const char* my_description; +public: + test_exception ( const char* description ) : my_description(description) {} + + const char* what() const throw() __TBB_override { return my_description; } +}; + +class solitary_test_exception : public test_exception { +public: + solitary_test_exception ( const char* description ) : test_exception(description) {} +}; + +#if TBB_USE_CAPTURED_EXCEPTION + typedef tbb::captured_exception PropagatedException; + #define EXCEPTION_NAME(e) e.name() +#else + typedef test_exception PropagatedException; + #define EXCEPTION_NAME(e) typeid(e).name() +#endif + +#define EXCEPTION_DESCR "Test exception" + +#if HARNESS_EH_SIMPLE_MODE + +static void ThrowTestException () { + ++g_ExceptionsThrown; + throw test_exception(EXCEPTION_DESCR); +} + +#else /* !HARNESS_EH_SIMPLE_MODE */ + +static void ThrowTestException ( intptr_t threshold ) { + bool inMaster = (Harness::CurrentTid() == g_Master); + if ( !g_ThrowException || // if we're not supposed to throw + (!g_Flog && // if we're not catching throw in bodies and + (g_ExceptionInMaster ^ inMaster)) ) { // we're the master and not expected to throw + // or are the master and the master is not the one to throw (??) + return; + } + while ( Existed() < threshold ) + __TBB_Yield(); + if ( !g_SolitaryException ) { + ++g_ExceptionsThrown; + if(inMaster) ++g_MasterExecutedThrow; else ++g_NonMasterExecutedThrow; + throw test_exception(EXCEPTION_DESCR); + } + // g_SolitaryException == true + if(g_NestedPipelines) { + // only throw exception if we have started at least two inner pipelines + // else return + if(g_PipelinesStarted >= 3) { + if ( g_ExceptionsThrown.compare_and_swap(1, 0) == 0 ) { + if(inMaster) ++g_MasterExecutedThrow; else ++g_NonMasterExecutedThrow; + throw solitary_test_exception(EXCEPTION_DESCR); + } + } + } + else { + if ( g_ExceptionsThrown.compare_and_swap(1, 0) == 0 ) { + if(inMaster) ++g_MasterExecutedThrow; else ++g_NonMasterExecutedThrow; + throw solitary_test_exception(EXCEPTION_DESCR); + } + } +} +#endif /* !HARNESS_EH_SIMPLE_MODE */ + +#define UPDATE_COUNTS() \ + { \ + ++g_CurExecuted; \ + if(g_Master == Harness::CurrentTid()) g_MasterExecuted = true; \ + else g_NonMasterExecuted = true; \ + if( tbb::task::self().is_cancelled() ) ++g_TGCCancelled; \ + } + +#define CATCH() \ + } catch ( PropagatedException& e ) { \ + g_ExecutedAtFirstCatch.compare_and_swap(g_CurExecuted,0); \ + g_ExecutedAtLastCatch = g_CurExecuted; \ + ASSERT( e.what(), "Empty what() string" ); \ + ASSERT (__TBB_EXCEPTION_TYPE_INFO_BROKEN || strcmp(EXCEPTION_NAME(e), (g_SolitaryException ? typeid(solitary_test_exception) : typeid(test_exception)).name() ) == 0, "Unexpected original exception name"); \ + ASSERT (__TBB_EXCEPTION_TYPE_INFO_BROKEN || strcmp(e.what(), EXCEPTION_DESCR) == 0, "Unexpected original exception info"); \ + g_ExceptionCaught = l_ExceptionCaughtAtCurrentLevel = true; \ + ++g_NumExceptionsCaught; \ + } catch ( tbb::tbb_exception& e ) { \ + REPORT("Unexpected %s\n", e.name()); \ + ASSERT (g_UnknownException && !g_UnknownException, "Unexpected tbb::tbb_exception" ); \ + } catch ( std::exception& e ) { \ + REPORT("Unexpected %s\n", typeid(e).name()); \ + ASSERT (g_UnknownException && !g_UnknownException, "Unexpected std::exception" ); \ + } catch ( ... ) { \ + g_ExceptionCaught = l_ExceptionCaughtAtCurrentLevel = true; \ + g_UnknownException = unknownException = true; \ + } \ + if ( !g_SolitaryException ) \ + REMARK_ONCE ("Multiple exceptions mode: %d throws", (intptr_t)g_ExceptionsThrown); + +#define ASSERT_EXCEPTION() \ + { \ + ASSERT (!g_ExceptionsThrown || g_ExceptionCaught, "throw without catch"); \ + ASSERT (!g_ExceptionCaught || g_ExceptionsThrown, "catch without throw"); \ + ASSERT (g_ExceptionCaught || (g_ExceptionInMaster && !g_MasterExecutedThrow) || (!g_ExceptionInMaster && !g_NonMasterExecutedThrow), "no exception occurred"); \ + ASSERT (__TBB_EXCEPTION_TYPE_INFO_BROKEN || !g_UnknownException, "unknown exception was caught"); \ + } + +#define CATCH_AND_ASSERT() \ + CATCH() \ + ASSERT_EXCEPTION() + +#else /* !TBB_USE_EXCEPTIONS */ + +inline void ThrowTestException ( intptr_t ) {} + +#endif /* !TBB_USE_EXCEPTIONS */ + +#define TRY() \ + bool l_ExceptionCaughtAtCurrentLevel = false, unknownException = false; \ + __TBB_TRY { + +// "l_ExceptionCaughtAtCurrentLevel || unknownException" is used only to "touch" otherwise unused local variables +#define CATCH_AND_FAIL() } __TBB_CATCH(...) { \ + ASSERT (false, "Cancelling tasks must not cause any exceptions"); \ + (void)(l_ExceptionCaughtAtCurrentLevel && unknownException); \ + } + +const int c_Timeout = 1000000; + +void WaitUntilConcurrencyPeaks ( int expected_peak ) { + if ( g_Flog ) + return; + int n = 0; +retry: + while ( ++n < c_Timeout && (int)Harness::ConcurrencyTracker::PeakParallelism() < expected_peak ) + __TBB_Yield(); +#if USE_TASK_SCHEDULER_OBSERVER + ASSERT_WARNING( g_NumThreads == g_ActualMaxThreads, "Library did not provide sufficient threads"); +#endif + ASSERT_WARNING(n < c_Timeout,g_Wakeup_Msg); + // Workaround in case a missed wakeup takes place + if ( n == c_Timeout ) { + tbb::task &r = *new( tbb::task::allocate_root() ) tbb::empty_task(); + r.spawn(r); + n = 0; + goto retry; + } +} + +inline void WaitUntilConcurrencyPeaks () { WaitUntilConcurrencyPeaks(g_NumThreads); } + +inline bool IsMaster() { + return Harness::CurrentTid() == g_Master; +} + +inline bool IsThrowingThread() { + return g_ExceptionInMaster ^ IsMaster() ? true : false; +} + +class CancellatorTask : public tbb::task { + static volatile bool s_Ready; + tbb::task_group_context &m_groupToCancel; + intptr_t m_cancellationThreshold; + + tbb::task* execute () __TBB_override { + Harness::ConcurrencyTracker ct; + s_Ready = true; + while ( g_CurExecuted < m_cancellationThreshold ) + __TBB_Yield(); + m_groupToCancel.cancel_group_execution(); + g_ExecutedAtLastCatch = g_CurExecuted; + return NULL; + } +public: + CancellatorTask ( tbb::task_group_context& ctx, intptr_t threshold ) + : m_groupToCancel(ctx), m_cancellationThreshold(threshold) + { + s_Ready = false; + } + + static void Reset () { s_Ready = false; } + + static bool WaitUntilReady () { + const intptr_t limit = 10000000; + intptr_t n = 0; + do { + __TBB_Yield(); + } while( !s_Ready && ++n < limit ); + // should yield once, then continue if Cancellator is ready. + ASSERT( s_Ready || n == limit, NULL ); + return s_Ready; + } +}; + +volatile bool CancellatorTask::s_Ready = false; + +template +void RunCancellationTest ( intptr_t threshold = 1 ) +{ + tbb::task_group_context ctx; + tbb::empty_task &r = *new( tbb::task::allocate_root(ctx) ) tbb::empty_task; + r.set_ref_count(3); + r.spawn( *new( r.allocate_child() ) CancellatorTaskT(ctx, threshold) ); + __TBB_Yield(); + r.spawn( *new( r.allocate_child() ) LauncherTaskT(ctx) ); + TRY(); + r.wait_for_all(); + CATCH_AND_FAIL(); + r.destroy(r); +} diff --git a/src/tbb/src/test/harness_fp.h b/src/tbb/src/test/harness_fp.h new file mode 100644 index 00000000..3fa9c174 --- /dev/null +++ b/src/tbb/src/test/harness_fp.h @@ -0,0 +1,168 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// include system header to prevent standard library to be included under private=public first time +#include +#define private public +#include "tbb/tbb_machine.h" +#undef private +#include "harness_assert.h" + +#if ( __TBB_x86_32 || __TBB_x86_64 ) && __TBB_CPU_CTL_ENV_PRESENT && !defined(__TBB_WIN32_USE_CL_BUILTINS) + +const int FE_TONEAREST = 0x0000, + FE_DOWNWARD = 0x0400, + FE_UPWARD = 0x0800, + FE_TOWARDZERO = 0x0c00, + FE_RND_MODE_MASK = FE_TOWARDZERO, + SSE_RND_MODE_MASK = FE_RND_MODE_MASK << 3, + SSE_DAZ = 0x0040, + SSE_FTZ = 0x8000, + SSE_MODE_MASK = SSE_DAZ | SSE_FTZ, + SSE_STATUS_MASK = 0x3F; + +const int NumSseModes = 4; +const int SseModes[NumSseModes] = { 0, SSE_DAZ, SSE_FTZ, SSE_DAZ | SSE_FTZ }; + +#if _WIN64 && !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE && !__MINGW64__ +// MinGW uses inline implementation from tbb/machine/linux_intel64.h +// and when inline asm is not available, the library uses out of line assembly which is not exported +// thus reimplementing them here + +#include + +inline void __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* fe ) { + fe->x87cw = short(_control87(0, 0) & _MCW_RC) << 2; + fe->mxcsr = _mm_getcsr(); +} +inline void __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* fe ) { + ASSERT( (fe->x87cw & FE_RND_MODE_MASK) == ((fe->x87cw & FE_RND_MODE_MASK) >> 2 & _MCW_RC) << 2, "Check float.h constants" ); + _control87( (fe->x87cw & FE_RND_MODE_MASK) >> 6, _MCW_RC ); + _mm_setcsr( fe->mxcsr ); +} + +#endif /* _WIN64 && !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE && !__MINGW64__ */ + +inline int GetRoundingMode ( bool checkConsistency = true ) { + tbb::internal::cpu_ctl_env ctl; + ctl.get_env(); + ASSERT( !checkConsistency || (ctl.mxcsr & SSE_RND_MODE_MASK) >> 3 == (ctl.x87cw & FE_RND_MODE_MASK), NULL ); + return ctl.x87cw & FE_RND_MODE_MASK; +} + +inline void SetRoundingMode ( int mode ) { + tbb::internal::cpu_ctl_env ctl; + ctl.get_env(); + ctl.mxcsr = (ctl.mxcsr & ~SSE_RND_MODE_MASK) | (mode & FE_RND_MODE_MASK) << 3; + ctl.x87cw = short((ctl.x87cw & ~FE_RND_MODE_MASK) | (mode & FE_RND_MODE_MASK)); + ctl.set_env(); +} + +inline int GetSseMode () { + tbb::internal::cpu_ctl_env ctl; + ctl.get_env(); + return ctl.mxcsr & SSE_MODE_MASK; +} + +inline void SetSseMode ( int mode ) { + tbb::internal::cpu_ctl_env ctl; + ctl.get_env(); + ctl.mxcsr = (ctl.mxcsr & ~SSE_MODE_MASK) | (mode & SSE_MODE_MASK); + ctl.set_env(); +} + +#elif defined(_M_ARM) || defined(__TBB_WIN32_USE_CL_BUILTINS) +const int NumSseModes = 1; +const int SseModes[NumSseModes] = { 0 }; + +inline int GetSseMode () { return 0; } +inline void SetSseMode ( int ) {} + +const int FE_TONEAREST = _RC_NEAR, + FE_DOWNWARD = _RC_DOWN, + FE_UPWARD = _RC_UP, + FE_TOWARDZERO = _RC_CHOP; + +inline int GetRoundingMode ( bool = true ) { + tbb::internal::cpu_ctl_env ctl; + ctl.get_env(); + return ctl.my_ctl; +} +inline void SetRoundingMode ( int mode ) { + tbb::internal::cpu_ctl_env ctl; + ctl.my_ctl = mode; + ctl.set_env(); +} + +#else /* Other archs */ + +#include + +const int RND_MODE_MASK = FE_TONEAREST | FE_DOWNWARD | FE_UPWARD | FE_TOWARDZERO; + +const int NumSseModes = 1; +const int SseModes[NumSseModes] = { 0 }; + +inline int GetRoundingMode ( bool = true ) { return fegetround(); } +inline void SetRoundingMode ( int rnd ) { fesetround(rnd); } + +inline int GetSseMode () { return 0; } +inline void SetSseMode ( int ) {} + +#endif /* Other archs */ + +const int NumRoundingModes = 4; +const int RoundingModes[NumRoundingModes] = { FE_TONEAREST, FE_DOWNWARD, FE_UPWARD, FE_TOWARDZERO }; +const int numFPModes = NumRoundingModes*NumSseModes; + +inline void SetFPMode( int mode ) { + SetRoundingMode( RoundingModes[mode/NumSseModes%NumRoundingModes] ); + SetSseMode( SseModes[mode%NumSseModes] ); +} + +#define AssertFPMode( mode ) { \ + ASSERT( GetRoundingMode() == RoundingModes[mode/NumSseModes%NumRoundingModes], "FPU control state has not been set correctly." ); \ + ASSERT( GetSseMode() == SseModes[mode%NumSseModes], "SSE control state has not been set correctly." ); \ +} + +inline int SetNextFPMode( int mode, int step = 1 ) { + const int nextMode = (mode+step)%numFPModes; + SetFPMode( nextMode ); + return nextMode; +} + +class FPModeContext { + int origSse, origRounding; + int currentMode; +public: + FPModeContext(int newMode) { + origSse = GetSseMode(); + origRounding = GetRoundingMode(); + SetFPMode(currentMode = newMode); + } + ~FPModeContext() { + assertFPMode(); + SetRoundingMode(origRounding); + SetSseMode(origSse); + } + int setNextFPMode() { + assertFPMode(); + return currentMode = SetNextFPMode(currentMode); + } + void assertFPMode() { + AssertFPMode(currentMode); + } +}; diff --git a/src/tbb/src/test/harness_graph.h b/src/tbb/src/test/harness_graph.h new file mode 100644 index 00000000..1ba4e694 --- /dev/null +++ b/src/tbb/src/test/harness_graph.h @@ -0,0 +1,1236 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/** @file harness_graph.cpp + This contains common helper classes and functions for testing graph nodes +**/ + +#ifndef harness_graph_H +#define harness_graph_H + +#include "harness.h" +#include "harness_barrier.h" +#include "tbb/flow_graph.h" +#include "tbb/null_rw_mutex.h" +#include "tbb/atomic.h" +#include "tbb/concurrent_unordered_map.h" +#include "tbb/task.h" +#include "tbb/task_scheduler_init.h" +#include "tbb/compat/condition_variable" +#include "tbb/mutex.h" +#include "tbb/tbb_thread.h" + +using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; + +#define WAIT_MAX 2000000 +#define BACKOFF_WAIT(ex,msg) \ +{ \ + int wait_cnt = 0; \ + tbb::internal::atomic_backoff backoff; \ + do { \ + backoff.pause(); \ + ++wait_cnt; \ + } \ + while( (ex) && (wait_cnt < WAIT_MAX)); \ + ASSERT(wait_cnt < WAIT_MAX, msg); \ +} +#define BACKOFF_WAIT_NOASSERT(ex,msg) \ +{ \ + int wait_cnt = 0; \ + tbb::internal::atomic_backoff backoff; \ + do { \ + backoff.pause(); \ + ++wait_cnt; \ + } \ + while( (ex) && (wait_cnt < WAIT_MAX)); \ + if(wait_cnt >= WAIT_MAX) REMARK("%s\n",msg); \ +} + +// Needed conversion to and from continue_msg, but didn't want to add +// conversion operators to the class, since we don't want it in general, +// only in these tests. +template +struct converter { + static OutputType convert_value(const InputType &i) { + return OutputType(i); + } +}; + +template +struct converter { + static tbb::flow::continue_msg convert_value(const InputType &/*i*/) { + return tbb::flow::continue_msg(); + } +}; + +template +struct converter { + static OutputType convert_value(const tbb::flow::continue_msg &/*i*/) { + return OutputType(); + } +}; + +// helper for multifunction_node tests. +template +struct mof_helper { + template + static inline void output_converted_value(const InputType &i, ports_type &p) { + (void)tbb::flow::get(p).try_put(converter::type::output_type>::convert_value(i)); + output_converted_value(i, p); + } +}; + +template<> +struct mof_helper<1> { + template + static inline void output_converted_value(const InputType &i, ports_type &p) { + // just emit a default-constructed object + (void)tbb::flow::get<0>(p).try_put(converter::type::output_type>::convert_value(i)); + } +}; + +template< typename InputType, typename OutputType > +struct harness_graph_default_functor { + static OutputType construct( InputType v ) { + return OutputType(v); + } +}; + +template< typename OutputType > +struct harness_graph_default_functor< tbb::flow::continue_msg, OutputType > { + static OutputType construct( tbb::flow::continue_msg ) { + return OutputType(); + } +}; + +template< typename InputType > +struct harness_graph_default_functor< InputType, tbb::flow::continue_msg > { + static tbb::flow::continue_msg construct( InputType ) { + return tbb::flow::continue_msg(); + } +}; + +template< > +struct harness_graph_default_functor< tbb::flow::continue_msg, tbb::flow::continue_msg > { + static tbb::flow::continue_msg construct( tbb::flow::continue_msg ) { + return tbb::flow::continue_msg(); + } +}; + +template +struct harness_graph_default_multifunction_functor { + static const int N = tbb::flow::tuple_size::value; + typedef typename tbb::flow::multifunction_node::output_ports_type ports_type; + static void construct(const InputType &i, ports_type &p) { + mof_helper::output_converted_value(i, p); + } +}; + +//! An executor that accepts InputType and generates OutputType +template< typename InputType, typename OutputType > +struct harness_graph_executor { + + typedef OutputType (*function_ptr_type)( InputType v ); + + template + struct mutex_holder { static RW mutex; }; + + static function_ptr_type fptr; + static tbb::atomic execute_count; + static tbb::atomic current_executors; + static size_t max_executors; + + static inline OutputType func( InputType v ) { + size_t c; // Declaration separate from initialization to avoid ICC internal error on IA-64 architecture + c = current_executors.fetch_and_increment(); + ASSERT( max_executors == 0 || c <= max_executors, NULL ); + ++execute_count; + OutputType v2 = (*fptr)(v); + current_executors.fetch_and_decrement(); + return v2; + } + + template< typename RW > + static inline OutputType tfunc( InputType v ) { + // Invocations allowed to be concurrent, the lock is acquired in shared ("read") mode. + // A test can take it exclusively, thus creating a barrier for invocations. + typename RW::scoped_lock l( mutex_holder::mutex, /*write=*/false ); + return func(v); + } + + template< typename RW > + struct tfunctor { + tbb::atomic my_execute_count; + tfunctor() { my_execute_count = 0; } + tfunctor( const tfunctor &f ) { my_execute_count = f.my_execute_count; } + OutputType operator()( InputType i ) { + typename RW::scoped_lock l( harness_graph_executor::mutex_holder::mutex, /*write=*/false ); + my_execute_count.fetch_and_increment(); + return harness_graph_executor::func(i); + } + }; + typedef tfunctor functor; + +}; + +//! A multifunction executor that accepts InputType and has only one Output of OutputType. +template< typename InputType, typename OutputTuple > +struct harness_graph_multifunction_executor { + typedef typename tbb::flow::multifunction_node::output_ports_type ports_type; + typedef typename tbb::flow::tuple_element<0,OutputTuple>::type OutputType; + + typedef void (*mfunction_ptr_type)( const InputType& v, ports_type &p ); + + template + struct mutex_holder { static RW mutex; }; + + static mfunction_ptr_type fptr; + static tbb::atomic execute_count; + static tbb::atomic current_executors; + static size_t max_executors; + + static inline void empty_func( const InputType&, ports_type& ) { + } + + static inline void func( const InputType &v, ports_type &p ) { + size_t c; // Declaration separate from initialization to avoid ICC internal error on IA-64 architecture + c = current_executors.fetch_and_increment(); + ASSERT( max_executors == 0 || c <= max_executors, NULL ); + ASSERT(tbb::flow::tuple_size::value == 1, NULL); + ++execute_count; + (*fptr)(v,p); + current_executors.fetch_and_decrement(); + } + + template< typename RW > + static inline void tfunc( const InputType& v, ports_type &p ) { + // Shared lock in invocations, exclusive in a test; see a comment in harness_graph_executor. + typename RW::scoped_lock l( mutex_holder::mutex, /*write=*/false ); + func(v,p); + } + + template< typename RW > + struct tfunctor { + tbb::atomic my_execute_count; + tfunctor() { my_execute_count = 0; } + tfunctor( const tfunctor &f ) { my_execute_count = f.my_execute_count; } + void operator()( const InputType &i, ports_type &p ) { + typename RW::scoped_lock l( harness_graph_multifunction_executor::mutex_holder::mutex, /*write=*/false ); + my_execute_count.fetch_and_increment(); + harness_graph_multifunction_executor::func(i,p); + } + }; + typedef tfunctor functor; + +}; + +// static vars for function_node tests +template< typename InputType, typename OutputType > +template< typename RW > +RW harness_graph_executor::mutex_holder::mutex; + +template< typename InputType, typename OutputType > +tbb::atomic harness_graph_executor::execute_count; + +template< typename InputType, typename OutputType > +typename harness_graph_executor::function_ptr_type harness_graph_executor::fptr + = harness_graph_default_functor< InputType, OutputType >::construct; + +template< typename InputType, typename OutputType > +tbb::atomic harness_graph_executor::current_executors; + +template< typename InputType, typename OutputType > +size_t harness_graph_executor::max_executors = 0; + +// static vars for multifunction_node tests +template< typename InputType, typename OutputTuple > +template< typename RW > +RW harness_graph_multifunction_executor::mutex_holder::mutex; + +template< typename InputType, typename OutputTuple > +tbb::atomic harness_graph_multifunction_executor::execute_count; + +template< typename InputType, typename OutputTuple > +typename harness_graph_multifunction_executor::mfunction_ptr_type harness_graph_multifunction_executor::fptr + = harness_graph_default_multifunction_functor< InputType, OutputTuple >::construct; + +template< typename InputType, typename OutputTuple > +tbb::atomic harness_graph_multifunction_executor::current_executors; + +template< typename InputType, typename OutputTuple > +size_t harness_graph_multifunction_executor::max_executors = 0; + +//! Counts the number of puts received +template< typename T > +struct harness_counting_receiver : public tbb::flow::receiver, NoAssign { + + tbb::atomic< size_t > my_count; + T max_value; + size_t num_copies; + tbb::flow::graph& my_graph; + + harness_counting_receiver(tbb::flow::graph& g) : num_copies(1), my_graph(g) { + my_count = 0; + } + + void initialize_map( const T& m, size_t c ) { + my_count = 0; + max_value = m; + num_copies = c; + } + + tbb::flow::graph& graph_reference() __TBB_override { + return my_graph; + } + + tbb::task *try_put_task( const T & ) __TBB_override { + ++my_count; + return const_cast(SUCCESSFULLY_ENQUEUED); + } + + void validate() { + size_t n = my_count; + ASSERT( n == num_copies*max_value, NULL ); + } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename tbb::flow::receiver::built_predecessors_type built_predecessors_type; + built_predecessors_type mbp; + built_predecessors_type &built_predecessors() __TBB_override { return mbp; } + typedef typename tbb::flow::receiver::predecessor_list_type predecessor_list_type; + typedef typename tbb::flow::receiver::predecessor_type predecessor_type; + void internal_add_built_predecessor(predecessor_type &) __TBB_override {} + void internal_delete_built_predecessor(predecessor_type &) __TBB_override {} + void copy_predecessors(predecessor_list_type &) __TBB_override { } + size_t predecessor_count() __TBB_override { return 0; } +#endif + void reset_receiver(tbb::flow::reset_flags /*f*/) __TBB_override { my_count = 0; } +}; + +//! Counts the number of puts received +template< typename T > +struct harness_mapped_receiver : public tbb::flow::receiver, NoCopy { + + tbb::atomic< size_t > my_count; + T max_value; + size_t num_copies; + typedef tbb::concurrent_unordered_map< T, tbb::atomic< size_t > > map_type; + map_type *my_map; + tbb::flow::graph& my_graph; + + harness_mapped_receiver(tbb::flow::graph& g) : my_map(NULL), my_graph(g) { + my_count = 0; + } + + ~harness_mapped_receiver() { + if ( my_map ) delete my_map; + } + + void initialize_map( const T& m, size_t c ) { + my_count = 0; + max_value = m; + num_copies = c; + if ( my_map ) delete my_map; + my_map = new map_type; + } + + tbb::task * try_put_task( const T &t ) __TBB_override { + if ( my_map ) { + tbb::atomic a; + a = 1; + std::pair< typename map_type::iterator, bool > r = (*my_map).insert( typename map_type::value_type( t, a ) ); + if ( r.second == false ) { + size_t v = r.first->second.fetch_and_increment(); + ASSERT( v < num_copies, NULL ); + } + } else { + ++my_count; + } + return const_cast(SUCCESSFULLY_ENQUEUED); + } + + tbb::flow::graph& graph_reference() __TBB_override { + return my_graph; + } + + void validate() { + if ( my_map ) { + for ( size_t i = 0; i < (size_t)max_value; ++i ) { + size_t n = (*my_map)[(int)i]; + ASSERT( n == num_copies, NULL ); + } + } else { + size_t n = my_count; + ASSERT( n == num_copies*max_value, NULL ); + } + } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename tbb::flow::receiver::built_predecessors_type built_predecessors_type; + built_predecessors_type mbp; + built_predecessors_type &built_predecessors() __TBB_override { return mbp; } + typedef typename tbb::flow::receiver::predecessor_list_type predecessor_list_type; + typedef typename tbb::flow::receiver::predecessor_type predecessor_type; + void internal_add_built_predecessor(predecessor_type &) __TBB_override {} + void internal_delete_built_predecessor(predecessor_type &) __TBB_override {} + void copy_predecessors(predecessor_list_type &) __TBB_override { } + size_t predecessor_count() __TBB_override { return 0; } +#endif + void reset_receiver(tbb::flow::reset_flags /*f*/) __TBB_override { + my_count = 0; + if(my_map) delete my_map; + my_map = new map_type; + } + +}; + +//! Counts the number of puts received +template< typename T > +struct harness_counting_sender : public tbb::flow::sender, NoCopy { + + typedef typename tbb::flow::sender::successor_type successor_type; + tbb::atomic< successor_type * > my_receiver; + tbb::atomic< size_t > my_count; + tbb::atomic< size_t > my_received; + size_t my_limit; + + harness_counting_sender( ) : my_limit(~size_t(0)) { + my_receiver = NULL; + my_count = 0; + my_received = 0; + } + + harness_counting_sender( size_t limit ) : my_limit(limit) { + my_receiver = NULL; + my_count = 0; + my_received = 0; + } + + bool register_successor( successor_type &r ) __TBB_override { + my_receiver = &r; + return true; + } + + bool remove_successor( successor_type &r ) __TBB_override { + successor_type *s = my_receiver.fetch_and_store( NULL ); + ASSERT( s == &r, NULL ); + return true; + } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename tbb::flow::sender::successor_list_type successor_list_type; + typedef typename tbb::flow::sender::built_successors_type built_successors_type; + built_successors_type bst; + built_successors_type &built_successors() __TBB_override { return bst; } + void internal_add_built_successor( successor_type &) __TBB_override {} + void internal_delete_built_successor( successor_type &) __TBB_override {} + void copy_successors(successor_list_type &) __TBB_override { } + size_t successor_count() __TBB_override { return 0; } +#endif + + bool try_get( T & v ) __TBB_override { + size_t i = my_count.fetch_and_increment(); + if ( i < my_limit ) { + v = T( i ); + ++my_received; + return true; + } else { + return false; + } + } + + bool try_put_once() { + successor_type *s = my_receiver; + size_t i = my_count.fetch_and_increment(); + if ( s->try_put( T(i) ) ) { + ++my_received; + return true; + } else { + return false; + } + } + + void try_put_until_false() { + successor_type *s = my_receiver; + size_t i = my_count.fetch_and_increment(); + + while ( s->try_put( T(i) ) ) { + ++my_received; + i = my_count.fetch_and_increment(); + } + } + + void try_put_until_limit() { + successor_type *s = my_receiver; + + for ( int i = 0; i < (int)my_limit; ++i ) { + ASSERT( s->try_put( T(i) ), NULL ); + ++my_received; + } + ASSERT( my_received == my_limit, NULL ); + } + +}; + +// test for resets of buffer-type nodes. +tbb::atomic serial_fn_state0; +tbb::atomic serial_fn_state1; +tbb::atomic serial_continue_state0; + +template +struct serial_fn_body { + tbb::atomic *_flag; + serial_fn_body(tbb::atomic &myatomic) : _flag(&myatomic) { } + T operator()(const T& in) { + if(*_flag == 0) { + *_flag = 1; + // wait until we are released + tbb::internal::atomic_backoff backoff; + do { + backoff.pause(); + } while(*_flag == 1); + } + // return value + return in; + } +}; + +template +struct serial_continue_body { + tbb::atomic *_flag; + serial_continue_body(tbb::atomic &myatomic) : _flag(&myatomic) {} + T operator()(const tbb::flow::continue_msg& /*in*/) { + // signal we have received a value + *_flag = 1; + // wait until we are released + tbb::internal::atomic_backoff backoff; + do { + backoff.pause(); + } while(*_flag == 1); + // return value + return (T)1; + } +}; + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + + +// walk two lists via iterator, match elements of each, in possibly-different ordder, and +// return true if all elements of sv appear in tv. +template +bool lists_match(SV &sv, TV &tv) { + if(sv.size() != tv.size()) { + return false; + } + std::vector bv(sv.size(), false); + for(typename TV::iterator itv = tv.begin(); itv != tv.end(); ++itv) { + int ibv = 0; + for(typename SV::iterator isv = sv.begin(); isv != sv.end(); ++isv) { + if(!bv[ibv]) { + if(*itv == *isv) { + bv[ibv] = true; + goto found_it;; + } + } + ++ibv; + } + return false; +found_it: + continue; + } + return true; +} +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ + +template +void test_resets() { + const int NN = 3; + tbb::task_scheduler_init init(4); + tbb::task_group_context tgc; + tbb::flow::graph g(tgc); + BufferType b0(g); + tbb::flow::queue_node q0(g); + T j; + bool nFound[NN]; + + // reset empties buffer + for(T i = 0; i < NN; ++i) { + b0.try_put(i); + nFound[(int)i] = false; + } + g.wait_for_all(); + g.reset(); + ASSERT(!b0.try_get(j), "reset did not empty buffer"); + + // reset doesn't delete edge + + tbb::flow::make_edge(b0,q0); + g.reset(); + for(T i = 0; i < NN; ++i) { + b0.try_put(i); + } + + g.wait_for_all(); + for( T i = 0; i < NN; ++i) { + ASSERT(q0.try_get(j), "Missing value from buffer"); + ASSERT(!nFound[(int)j], "Duplicate value found"); + nFound[(int)j] = true; + } + + for(int ii = 0; ii < NN; ++ii) { + ASSERT(nFound[ii], "missing value"); + } + ASSERT(!q0.try_get(j), "Extra values in output"); + + // reset reverses a reversed edge. + // we will use a serial rejecting node to get the edge to reverse. + tbb::flow::function_node sfn(g, tbb::flow::serial, serial_fn_body(serial_fn_state0)); + tbb::flow::queue_node outq(g); + tbb::flow::remove_edge(b0,q0); + tbb::flow::make_edge(b0, sfn); + tbb::flow::make_edge(sfn,outq); + g.wait_for_all(); // wait for all the tasks started by building the graph are done. + serial_fn_state0 = 0; + + // b0 ------> sfn ------> outq + + for(int icnt = 0; icnt < 2; ++icnt) { + g.wait_for_all(); + serial_fn_state0 = 0; + b0.try_put((T)0); // will start sfn + // wait until function_node starts + BACKOFF_WAIT(serial_fn_state0 == 0,"Timed out waiting for function_node to start"); + // now the function_node is executing. + // this will start a task to forward the second item + // to the serial function node + b0.try_put((T)1); // first item will be consumed by task completing the execution + BACKOFF_WAIT_NOASSERT(g.root_task()->ref_count() >= 3,"Timed out waiting try_put task to wind down"); + b0.try_put((T)2); // second item will remain after cancellation + // now wait for the task that attempts to forward the buffer item to + // complete. + BACKOFF_WAIT_NOASSERT(g.root_task()->ref_count() >= 3,"Timed out waiting for tasks to wind down"); + // now cancel the graph. + ASSERT(tgc.cancel_group_execution(), "task group already cancelled"); + serial_fn_state0 = 0; // release the function_node. + g.wait_for_all(); // wait for all the tasks to complete. + // check that at most one output reached the queue_node + T outt; + T outt2; + bool got_item1 = outq.try_get(outt); + bool got_item2 = outq.try_get(outt2); + // either the output queue was empty (if the function_node tested for cancellation before putting the + // result to the queue) or there was one element in the queue (the 0). + ASSERT(!got_item1 || ((int)outt == 0 && !got_item2), "incorrect output from function_node"); + // the edge between the buffer and the function_node should be reversed, and the last + // message we put in the buffer should still be there. We can't directly test for the + // edge reversal. + got_item1 = b0.try_get(outt); + ASSERT(got_item1, " buffer lost a message"); + ASSERT(2 == (int)outt || 1 == (int)outt, " buffer had incorrect message"); // the one not consumed by the node. + ASSERT(g.is_cancelled(), "Graph was not cancelled"); + g.reset(); + } // icnt + + // reset with remove_edge removes edge. (icnt ==0 => forward edge, 1 => reversed edge + for(int icnt = 0; icnt < 2; ++icnt) { + if(icnt == 1) { + // set up reversed edge + tbb::flow::make_edge(b0, sfn); + tbb::flow::make_edge(sfn,outq); + serial_fn_state0 = 0; + b0.try_put((T)0); // starts up the function node + b0.try_put((T)1); // shoyuld reverse the edge + BACKOFF_WAIT(serial_fn_state0 == 0,"Timed out waiting for edge reversal"); + ASSERT(tgc.cancel_group_execution(), "task group already cancelled"); + serial_fn_state0 = 0; // release the function_node. + g.wait_for_all(); // wait for all the tasks to complete. + } + g.reset(tbb::flow::rf_clear_edges); + // test that no one is a successor to the buffer now. + serial_fn_state0 = 1; // let the function_node go if it gets an input message + b0.try_put((T)23); + g.wait_for_all(); + ASSERT((int)serial_fn_state0 == 1, "function_node executed when it shouldn't"); + T outt; + ASSERT(b0.try_get(outt) && (T)23 == outt, "node lost its input"); + } +} + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + +template< typename NODE_TYPE > +class test_buffer_base_extract { +protected: + tbb::flow::graph &g; + NODE_TYPE &in0; + NODE_TYPE &in1; + NODE_TYPE &middle; + NODE_TYPE &out0; + NODE_TYPE &out1; + NODE_TYPE *ins[2]; + NODE_TYPE *outs[2]; + typename NODE_TYPE::successor_type *ms_ptr; + typename NODE_TYPE::predecessor_type *mp_ptr; + + typename NODE_TYPE::predecessor_list_type in0_p_list; + typename NODE_TYPE::successor_list_type in0_s_list; + typename NODE_TYPE::predecessor_list_type in1_p_list; + typename NODE_TYPE::successor_list_type in1_s_list; + typename NODE_TYPE::predecessor_list_type out0_p_list; + typename NODE_TYPE::successor_list_type out0_s_list; + typename NODE_TYPE::predecessor_list_type out1_p_list; + typename NODE_TYPE::successor_list_type out1_s_list; + typename NODE_TYPE::predecessor_list_type mp_list; + typename NODE_TYPE::predecessor_list_type::iterator mp_list_iter; + typename NODE_TYPE::successor_list_type ms_list; + typename NODE_TYPE::successor_list_type::iterator ms_list_iter; + + virtual void set_up_lists() { + in0_p_list.clear(); + in0_s_list.clear(); + in1_p_list.clear(); + in1_s_list.clear(); + mp_list.clear(); + ms_list.clear(); + out0_p_list.clear(); + out0_s_list.clear(); + out1_p_list.clear(); + out1_s_list.clear(); + in0.copy_predecessors(in0_p_list); + in0.copy_successors(in0_s_list); + in1.copy_predecessors(in1_p_list); + in1.copy_successors(in1_s_list); + middle.copy_predecessors(mp_list); + middle.copy_successors(ms_list); + out0.copy_predecessors(out0_p_list); + out0.copy_successors(out0_s_list); + out1.copy_predecessors(out1_p_list); + out1.copy_successors(out1_s_list); + } + + void make_and_validate_full_graph() { + /* in0 out0 */ + /* \ / */ + /* middle */ + /* / \ */ + /* in1 out1 */ + tbb::flow::make_edge( in0, middle ); + tbb::flow::make_edge( in1, middle ); + tbb::flow::make_edge( middle, out0 ); + tbb::flow::make_edge( middle, out1 ); + + set_up_lists(); + + ASSERT( in0.predecessor_count() == 0 && in0_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in0.successor_count() == 1 && in0_s_list.size() == 1 && *(in0_s_list.begin()) == ms_ptr, "expected 1 successor" ); + ASSERT( in1.predecessor_count() == 0 && in1_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in1.successor_count() == 1 && in1_s_list.size() == 1 && *(in1_s_list.begin()) == ms_ptr, "expected 1 successor" ); + ASSERT( middle.predecessor_count() == 2 && mp_list.size() == 2, "expected 2 predecessors" ); + ASSERT( middle.successor_count() == 2 && ms_list.size() == 2, "expected 2 successors" ); + ASSERT( out0.predecessor_count() == 1 && out0_p_list.size() == 1 && *(out0_p_list.begin()) == mp_ptr, "expected 1 predecessor" ); + ASSERT( out0.successor_count() == 0 && out0_s_list.size() == 0, "expected 0 successors" ); + ASSERT( out1.predecessor_count() == 1 && out1_p_list.size() == 1 && *(out1_p_list.begin()) == mp_ptr, "expected 1 predecessor" ); + ASSERT( out1.successor_count() == 0 && out1_s_list.size() == 0, "expected 0 successors" ); + + int first_pred = *(mp_list.begin()) == ins[0] ? 0 : ( *(mp_list.begin()) == ins[1] ? 1 : -1 ); + mp_list_iter = mp_list.begin(); ++mp_list_iter; + int second_pred = *mp_list_iter == ins[0] ? 0 : ( *mp_list_iter == ins[1] ? 1 : -1 ); + ASSERT( first_pred != -1 && second_pred != -1 && first_pred != second_pred, "bad predecessor(s) for middle" ); + + int first_succ = *(ms_list.begin()) == outs[0] ? 0 : ( *(ms_list.begin()) == outs[1] ? 1 : -1 ); + ms_list_iter = ++(ms_list.begin()); + int second_succ = *ms_list_iter == outs[0] ? 0 : ( *ms_list_iter == outs[1] ? 1 : -1 ); + ASSERT( first_succ != -1 && second_succ != -1 && first_succ != second_succ, "bad successor(s) for middle" ); + + in0.try_put(1); + in1.try_put(2); + g.wait_for_all(); + + int r = 0; + int v = 0; + + ASSERT( in0.try_get(v) == false, "buffer should not have a value" ); + ASSERT( in1.try_get(v) == false, "buffer should not have a value" ); + ASSERT( middle.try_get(v) == false, "buffer should not have a value" ); + while ( out0.try_get(v) ) { + ASSERT( (v == 1 || v == 2) && (v&r) == 0, "duplicate value" ); + r |= v; + g.wait_for_all(); + } + while ( out1.try_get(v) ) { + ASSERT( (v == 1 || v == 2) && (v&r) == 0, "duplicate value" ); + r |= v; + g.wait_for_all(); + } + ASSERT( r == 3, "not all values received" ); + g.wait_for_all(); + } + + void validate_half_graph() { + /* in0 out0 */ + /* */ + /* middle */ + /* / \ */ + /* in1 out1 */ + set_up_lists(); + + ASSERT( in0.predecessor_count() == 0 && in0_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in0.successor_count() == 0 && in0_s_list.size() == 0, "expected 0 successors" ); + ASSERT( in1.predecessor_count() == 0 && in1_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in1.successor_count() == 1 && in1_s_list.size() == 1 && *(in1_s_list.begin()) == ms_ptr, "expected 1 successor" ); + ASSERT( middle.predecessor_count() == 1 && mp_list.size() == 1, "expected 1 predecessor" ); + ASSERT( middle.successor_count() == 1 && ms_list.size() == 1, "expected 1 successor" ); + ASSERT( out0.predecessor_count() == 0 && out0_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( out0.successor_count() == 0 && out0_s_list.size() == 0, "expected 0 successors" ); + ASSERT( out1.predecessor_count() == 1 && out1_p_list.size() == 1 && *(out1_p_list.begin()) == mp_ptr, "expected 1 predecessor" ); + ASSERT( out1.successor_count() == 0 && out1_s_list.size() == 0, "expected 0 successors" ); + + ASSERT( middle.predecessor_count() == 1 && mp_list.size() == 1, "expected two predecessors" ); + ASSERT( middle.successor_count() == 1 && ms_list.size() == 1, "expected two successors" ); + + ASSERT( *(mp_list.begin()) == ins[1], "incorrect predecessor" ); + ASSERT( *(ms_list.begin()) == outs[1], "incorrect successor" ); + + in0.try_put(1); + in1.try_put(2); + g.wait_for_all(); + + int v = 0; + ASSERT( in0.try_get(v) == true && v == 1, "buffer should have a value of 1" ); + ASSERT( in1.try_get(v) == false, "buffer should not have a value" ); + ASSERT( middle.try_get(v) == false, "buffer should not have a value" ); + ASSERT( out0.try_get(v) == false, "buffer should not have a value" ); + ASSERT( out1.try_get(v) == true && v == 2, "buffer should have a value of 2" ); + g.wait_for_all(); + } + + void validate_empty_graph() { + /* in0 out0 */ + /* */ + /* middle */ + /* */ + /* in1 out1 */ + set_up_lists(); + + ASSERT( in0.predecessor_count() == 0 && in0_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in0.successor_count() == 0 && in0_s_list.size() == 0, "expected 0 successors" ); + ASSERT( in1.predecessor_count() == 0 && in1_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( in1.successor_count() == 0 && in1_s_list.size() == 0, "expected 0 successors" ); + ASSERT( middle.predecessor_count() == 0 && mp_list.size() == 0, "expected 0 predecessors" ); + ASSERT( middle.successor_count() == 0 && ms_list.size() == 0, "expected 0 successors" ); + ASSERT( out0.predecessor_count() == 0 && out0_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( out0.successor_count() == 0 && out0_s_list.size() == 0, "expected 0 successors" ); + ASSERT( out1.predecessor_count() == 0 && out1_p_list.size() == 0, "expected 0 predecessors" ); + ASSERT( out1.successor_count() == 0 && out1_s_list.size() == 0, "expected 0 successors" ); + + ASSERT( middle.predecessor_count() == 0 && mp_list.size() == 0, "expected 0 predecessors" ); + ASSERT( middle.successor_count() == 0 && ms_list.size() == 0, "expected 0 successors" ); + + in0.try_put(1); + in1.try_put(2); + g.wait_for_all(); + + int v = 0; + ASSERT( in0.try_get(v) == true && v == 1, "buffer should have a value of 1" ); + ASSERT( in1.try_get(v) == true && v == 2, "buffer should have a value of 2" ); + ASSERT( middle.try_get(v) == false, "buffer should not have a value" ); + ASSERT( out0.try_get(v) == false, "buffer should not have a value" ); + ASSERT( out1.try_get(v) == false, "buffer should not have a value" ); + g.wait_for_all(); + } + + // forbid the ecompiler generation of operator= (VS2012 warning) + test_buffer_base_extract& operator=(test_buffer_base_extract & /*other*/); + +public: + + test_buffer_base_extract(tbb::flow::graph &_g, NODE_TYPE &i0, NODE_TYPE &i1, NODE_TYPE &m, NODE_TYPE &o0, NODE_TYPE &o1) : + g(_g), in0(i0), in1(i1), middle(m), out0(o0), out1(o1) { + ins[0] = &in0; + ins[1] = &in1; + outs[0] = &out0; + outs[1] = &out1; + ms_ptr = static_cast< typename NODE_TYPE::successor_type * >(&middle); + mp_ptr = static_cast< typename NODE_TYPE::predecessor_type *>(&middle); + } + + virtual ~test_buffer_base_extract() {} + + void run_tests() { + make_and_validate_full_graph(); + + in0.extract(); + out0.extract(); + validate_half_graph(); + + in1.extract(); + out1.extract(); + validate_empty_graph(); + + make_and_validate_full_graph(); + + middle.extract(); + validate_empty_graph(); + + make_and_validate_full_graph(); + } + +}; + +template< typename NODE_TYPE > +class test_buffer_extract : public test_buffer_base_extract { +protected: + tbb::flow::graph my_g; + NODE_TYPE my_in0; + NODE_TYPE my_in1; + NODE_TYPE my_middle; + NODE_TYPE my_out0; + NODE_TYPE my_out1; +public: + test_buffer_extract() : test_buffer_base_extract( my_g, my_in0, my_in1, my_middle, my_out0, my_out1), + my_in0(my_g), my_in1(my_g), my_middle(my_g), my_out0(my_g), my_out1(my_g) { } +}; + +template< > +class test_buffer_extract< tbb::flow::sequencer_node > : public test_buffer_base_extract< tbb::flow::sequencer_node > { +protected: + typedef tbb::flow::sequencer_node my_node_t; + tbb::flow::graph my_g; + my_node_t my_in0; + my_node_t my_in1; + my_node_t my_middle; + my_node_t my_out0; + my_node_t my_out1; + + typedef tbb::atomic count_t; + count_t middle_count; + count_t out0_count; + count_t out1_count; + + struct always_zero { size_t operator()(int) { return 0; } }; + struct always_inc { + count_t *c; + always_inc(count_t &_c) : c(&_c) {} + size_t operator()(int) { + return c->fetch_and_increment(); + } + }; + + void set_up_lists() __TBB_override { + middle_count = 0; + out0_count = 0; + out1_count = 0; + my_g.reset(); // reset the sequencer nodes to start at 0 again + test_buffer_base_extract< my_node_t >::set_up_lists(); + } + + +public: + test_buffer_extract() : test_buffer_base_extract( my_g, my_in0, my_in1, my_middle, my_out0, my_out1), + my_in0(my_g, always_zero()), my_in1(my_g, always_zero()), my_middle(my_g, always_inc(middle_count)), + my_out0(my_g, always_inc(out0_count)), my_out1(my_g, always_inc(out1_count)) { + } +}; + +// test for simple node that has one input, one output (overwrite_node, write_once_node, limiter_node) +// decrement tests have to be done separately. +template class NType, typename ItemType> +void test_extract_on_node() { + tbb::flow::graph g; + ItemType dont_care; + NType node0(g); + tbb::flow::queue_node q0(g); + tbb::flow::queue_node q1(g); + tbb::flow::queue_node q2(g); + for( int i = 0; i < 2; ++i) { + tbb::flow::make_edge(q0,node0); + tbb::flow::make_edge(q1,node0); + tbb::flow::make_edge(node0, q2); + q0.try_put(ItemType(i)); + g.wait_for_all(); + + /* q0 */ + /* \ */ + /* \ */ + /* node0 -- q2 */ + /* / */ + /* / */ + /* q1 */ + + ASSERT(node0.predecessor_count() == 2 && q0.successor_count() == 1 && q1.successor_count() == 1, "bad predecessor count"); + ASSERT(node0.successor_count() == 1 && q2.predecessor_count() == 1, "bad successor count"); + + ASSERT(q2.try_get(dont_care) && int(dont_care) == i, "item not forwarded"); + typename NType::successor_list_type sv, sv1; + typename NType::predecessor_list_type pv, pv1; + + pv1.push_back(&q0); + pv1.push_back(&q1); + sv1.push_back(&q2); + node0.copy_predecessors(pv); + node0.copy_successors(sv); + ASSERT(lists_match(pv,pv1), "predecessor vector incorrect"); + ASSERT(lists_match(sv,sv1), "successor vector incorrect"); + + if(i == 0) { + node0.extract(); + } + else { + q0.extract(); + q1.extract(); + q2.extract(); + } + + q0.try_put(ItemType(2)); + g.wait_for_all(); + ASSERT(!q2.try_get(dont_care), "node0 not disconnected"); + ASSERT(q0.try_get(dont_care), "q0 empty (should have one item)"); + + node0.copy_predecessors(pv); + node0.copy_successors(sv); + ASSERT(node0.predecessor_count() == 0 && q0.successor_count() == 0 && q1.successor_count() == 0, "error in pred count after extract"); + ASSERT(pv.size() == 0, "error in pred array count after extract"); + ASSERT(node0.successor_count() == 0 && q2.predecessor_count() == 0, "error in succ count after extract"); + ASSERT(sv.size() == 0, "error in succ array count after extract"); + g.wait_for_all(); + } +} + +#endif // TBB_DEPRECATED_FLOW_NODE_EXTRACTION + +template +void test_input_ports_return_ref(NodeType& mip_node) { + typename NodeType::input_ports_type& input_ports1 = mip_node.input_ports(); + typename NodeType::input_ports_type& input_ports2 = mip_node.input_ports(); + ASSERT(&input_ports1 == &input_ports2, "input_ports() should return reference"); +} + +template +void test_output_ports_return_ref(NodeType& mop_node) { + typename NodeType::output_ports_type& output_ports1 = mop_node.output_ports(); + typename NodeType::output_ports_type& output_ports2 = mop_node.output_ports(); + ASSERT(&output_ports1 == &output_ports2, "output_ports() should return reference"); +} + +template< template class ReservingNodeType, typename DataType, bool DoClear > +class harness_reserving_body : NoAssign { + ReservingNodeType &my_reserving_node; + tbb::flow::buffer_node &my_buffer_node; +public: + harness_reserving_body(ReservingNodeType &reserving_node, tbb::flow::buffer_node &bn) : my_reserving_node(reserving_node), my_buffer_node(bn) {} + void operator()(DataType i) const { + my_reserving_node.try_put(i); +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if (DoClear) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + my_reserving_node.clear(); + } + my_buffer_node.try_put(i); + my_reserving_node.try_put(i); + } +}; + +template< template class ReservingNodeType, typename DataType > +void test_reserving_nodes() { + const size_t N = 300; + + tbb::flow::graph g; + + ReservingNodeType reserving_n(g); + + tbb::flow::buffer_node buffering_n(g); + tbb::flow::join_node< tbb::flow::tuple, tbb::flow::reserving > join_n(g); + harness_counting_receiver< tbb::flow::tuple > end_receiver(g); + + tbb::flow::make_edge(reserving_n, tbb::flow::input_port<0>(join_n)); + tbb::flow::make_edge(buffering_n, tbb::flow::input_port<1>(join_n)); + tbb::flow::make_edge(join_n, end_receiver); + + NativeParallelFor(N, harness_reserving_body(reserving_n, buffering_n)); + g.wait_for_all(); + + ASSERT(end_receiver.my_count == N, NULL); + + // Should not hang + NativeParallelFor(N, harness_reserving_body(reserving_n, buffering_n)); + g.wait_for_all(); + + ASSERT(end_receiver.my_count == 2 * N, NULL); +} + +namespace lightweight_testing { + +typedef tbb::flow::tuple output_tuple_type; + +template +class native_loop_body : NoAssign { + NodeType& my_node; +public: + native_loop_body(NodeType& node) : my_node(node) {} + + void operator()(int) const { + tbb::tbb_thread::id this_id = tbb::this_tbb_thread::get_id(); + my_node.try_put(this_id); + } +}; + +class concurrency_checker_body { +public: + tbb::atomic my_body_count; + + concurrency_checker_body() { + my_body_count = 0; + } + + template + void operator()(const tbb::tbb_thread::id& input, gateway_type&) { + increase_and_check(input); + } + + output_tuple_type operator()(const tbb::tbb_thread::id& input) { + increase_and_check(input); + return output_tuple_type(); + } + +private: + void increase_and_check(const tbb::tbb_thread::id& input) { + ++my_body_count; + tbb::tbb_thread::id body_thread_id = tbb::this_tbb_thread::get_id(); + ASSERT(input == body_thread_id, "Body executed as not lightweight"); + } +}; + +template +void test_unlimited_lightweight_execution(unsigned N) { + tbb::flow::graph g; + NodeType node(g, tbb::flow::unlimited, concurrency_checker_body()); + + NativeParallelFor(N, native_loop_body(node)); + g.wait_for_all(); + + concurrency_checker_body body = tbb::flow::copy_body(node); + ASSERT(body.my_body_count == N, "Body needs to be executed N times"); +} + +// Using TBB implementation of condition variable +// not to include std header, which has problems with old GCC +using tbb::interface5::condition_variable; +using tbb::interface5::unique_lock; + +tbb::mutex m; +condition_variable lightweight_condition; +bool work_submitted; +bool lightweight_work_processed; + +template +class native_loop_limited_body : NoAssign { + NodeType& my_node; + Harness::SpinBarrier& my_barrier; +public: + native_loop_limited_body(NodeType& node, Harness::SpinBarrier& barrier): + my_node(node), my_barrier(barrier) {} + void operator()(int) const { + tbb::tbb_thread::id this_id = tbb::this_tbb_thread::get_id(); + my_node.try_put(this_id); + if(!lightweight_work_processed) { + my_barrier.wait(); + work_submitted = true; + lightweight_condition.notify_all(); + } + } +}; + +struct condition_predicate { + bool operator()() { + return work_submitted; + } +}; + +class limited_lightweight_checker_body { +public: + tbb::atomic my_body_count; + tbb::atomic my_lightweight_count; + tbb::atomic my_task_count; + limited_lightweight_checker_body() { + my_body_count = 0; + my_lightweight_count = 0; + my_task_count = 0; + } +private: + void increase_and_check(const tbb::tbb_thread::id& /*input*/) { + ++my_body_count; + bool is_task = tbb::task::self().state() == tbb::task::executing; + if(is_task) { + ++my_task_count; + } else { + unique_lock lock(m); + lightweight_condition.wait(lock, condition_predicate()); + ++my_lightweight_count; + lightweight_work_processed = true; + } + } +public: + template + void operator()(const tbb::tbb_thread::id& input, gateway_type&) { + increase_and_check(input); + } + output_tuple_type operator()(const tbb::tbb_thread::id& input) { + increase_and_check(input); + return output_tuple_type(); + } +}; + +template +void test_limited_lightweight_execution(unsigned N, unsigned concurrency) { + ASSERT(concurrency != tbb::flow::unlimited, + "Test for limited concurrency cannot be called with unlimited concurrency argument"); + tbb::flow::graph g; + NodeType node(g, concurrency, limited_lightweight_checker_body()); + // Execute first body as lightweight, then wait for all other threads to fill internal buffer. + // Then unblock the lightweightd thread and check if other body executions are inside tbb task. + Harness::SpinBarrier barrier(N - concurrency); + NativeParallelFor(N, native_loop_limited_body(node, barrier)); + g.wait_for_all(); + limited_lightweight_checker_body body = tbb::flow::copy_body(node); + ASSERT(body.my_body_count == N, "Body needs to be executed N times"); + ASSERT(body.my_lightweight_count == concurrency, "Body needs to be executed as lightweight once"); + ASSERT(body.my_task_count == N - concurrency, "Body needs to be executed as not lightweight N - 1 times"); + work_submitted = false; + lightweight_work_processed = false; +} + +template +void test_lightweight(unsigned N) { + test_unlimited_lightweight_execution(N); + test_limited_lightweight_execution(N, tbb::flow::serial); + test_limited_lightweight_execution(N, (std::min)(tbb::tbb_thread::hardware_concurrency() / 2, N/2)); +} + +template class NodeType> +void test(unsigned N) { + typedef tbb::tbb_thread::id input_type; + typedef tbb::cache_aligned_allocator allocator_type; + typedef NodeType node_type; + test_lightweight(N); +} + +} + +#endif diff --git a/src/tbb/src/test/harness_inject_scheduler.h b/src/tbb/src/test/harness_inject_scheduler.h new file mode 100644 index 00000000..804efed0 --- /dev/null +++ b/src/tbb/src/test/harness_inject_scheduler.h @@ -0,0 +1,82 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Used in tests that work with TBB scheduler but do not link to the TBB library. +// In other words it embeds the TBB library core into the test executable. + +#ifndef harness_inject_scheduler_H +#define harness_inject_scheduler_H + +#if HARNESS_DEFINE_PRIVATE_PUBLIC +#include // merely prevents LNK2019 error to happen (on ICL+VC9 configurations) +#include // include it first to avoid error on define below +#define private public +#define protected public +#endif + +// Suppress usage of #pragma comment +#define __TBB_NO_IMPLICIT_LINKAGE 1 + +// Enable preview features if any +#define __TBB_BUILD 1 + +#undef DO_ITT_NOTIFY + +#define __TBB_SOURCE_DIRECTLY_INCLUDED 1 +#include "../tbb/tbb_main.cpp" +#include "../tbb/dynamic_link.cpp" +#include "../tbb/tbb_misc_ex.cpp" + +// Tasking subsystem files +#include "../tbb/governor.cpp" +#include "../tbb/market.cpp" +#include "../tbb/arena.cpp" +#include "../tbb/scheduler.cpp" +#include "../tbb/observer_proxy.cpp" +#include "../tbb/task.cpp" +#include "../tbb/task_group_context.cpp" + +// Other dependencies +#include "../tbb/cache_aligned_allocator.cpp" +#include "../tbb/tbb_thread.cpp" +#include "../tbb/mutex.cpp" +#include "../tbb/spin_rw_mutex.cpp" +#include "../tbb/spin_mutex.cpp" +#include "../tbb/private_server.cpp" +#include "../tbb/concurrent_monitor.cpp" +#if _WIN32||_WIN64 +#include "../tbb/semaphore.cpp" +#endif +#include "../rml/client/rml_tbb.cpp" + +#if HARNESS_USE_RUNTIME_LOADER +#undef HARNESS_USE_RUNTIME_LOADER +#include "harness.h" + +int TestMain () { + // Tests that directly include sources make no sense in runtime loader testing mode. + return Harness::Skipped; +} +// Renaming the TestMain function avoids conditional compilation around same function in the test file +#define TestMain TestMainSkipped +#endif + +#if HARNESS_DEFINE_PRIVATE_PUBLIC +#undef protected +#undef private +#endif + +#endif /* harness_inject_scheduler_H */ diff --git a/src/tbb/src/test/harness_iterator.h b/src/tbb/src/test/harness_iterator.h new file mode 100644 index 00000000..faff6953 --- /dev/null +++ b/src/tbb/src/test/harness_iterator.h @@ -0,0 +1,160 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef harness_iterator_H +#define harness_iterator_H + +#include +#include +#include "tbb/atomic.h" +#include "harness_assert.h" + +namespace Harness { + +template +class InputIterator { +public: + typedef std::input_iterator_tag iterator_category; + typedef T value_type; + typedef typename std::allocator::difference_type difference_type; + typedef typename std::allocator::pointer pointer; + typedef typename std::allocator::reference reference; + + explicit InputIterator ( T * ptr ) : my_ptr(ptr), my_shared_epoch(new Epoch), my_current_epoch(0) {} + + InputIterator( const InputIterator& it ) { + ASSERT(it.my_current_epoch == it.my_shared_epoch->epoch, "Copying an invalidated iterator"); + my_ptr = it.my_ptr; + my_shared_epoch = it.my_shared_epoch; + my_current_epoch = it.my_current_epoch; + ++my_shared_epoch->refcounter; + } + + InputIterator& operator= ( const InputIterator& it ) { + ASSERT(it.my_current_epoch == it.my_shared_epoch->epoch, "Assigning an invalidated iterator"); + my_ptr = it.my_ptr; + my_current_epoch = it.my_current_epoch; + if(my_shared_epoch == it.my_shared_epoch) + return *this; + destroy(); + my_shared_epoch = it.my_shared_epoch; + ++my_shared_epoch->refcounter; + return *this; + } + + T& operator* () const { + ASSERT(my_shared_epoch->epoch == my_current_epoch, "Dereferencing an invalidated input iterator"); + return *my_ptr; + } + + InputIterator& operator++ () { + ASSERT(my_shared_epoch->epoch == my_current_epoch, "Incrementing an invalidated input iterator"); + ++my_ptr; + ++my_current_epoch; + ++my_shared_epoch->epoch; + return *this; + } + + bool operator== ( const InputIterator& it ) const { + ASSERT(my_shared_epoch->epoch == my_current_epoch, "Comparing an invalidated input iterator"); + ASSERT(it.my_shared_epoch->epoch == it.my_current_epoch, "Comparing with an invalidated input iterator"); + return my_ptr == it.my_ptr; + } + + ~InputIterator() { + destroy(); + } +private: + void destroy() { + if(0 == --my_shared_epoch->refcounter) { + delete my_shared_epoch; + } + } + struct Epoch { + typedef tbb::atomic Counter; + Epoch() { epoch = 0; refcounter = 1; } + Counter epoch; + Counter refcounter; + }; + + T * my_ptr; + Epoch *my_shared_epoch; + size_t my_current_epoch; +}; + +template +class ForwardIterator { + T * my_ptr; +public: + typedef std::forward_iterator_tag iterator_category; + typedef T value_type; + typedef typename std::allocator::difference_type difference_type; + typedef typename std::allocator::pointer pointer; + typedef typename std::allocator::reference reference; + + explicit ForwardIterator ( T * ptr ) : my_ptr(ptr){} + + ForwardIterator ( const ForwardIterator& r ) : my_ptr(r.my_ptr){} + T& operator* () const { return *my_ptr; } + ForwardIterator& operator++ () { ++my_ptr; return *this; } + bool operator== ( const ForwardIterator& r ) const { return my_ptr == r.my_ptr; } +}; + +template +class RandomIterator { + T * my_ptr; +public: + typedef std::random_access_iterator_tag iterator_category; + typedef T value_type; + typedef typename std::allocator::pointer pointer; + typedef typename std::allocator::reference reference; + typedef typename std::allocator::difference_type difference_type; + + explicit RandomIterator ( T * ptr ) : my_ptr(ptr){} + RandomIterator ( const RandomIterator& r ) : my_ptr(r.my_ptr){} + T& operator* () const { return *my_ptr; } + RandomIterator& operator++ () { ++my_ptr; return *this; } + bool operator== ( const RandomIterator& r ) const { return my_ptr == r.my_ptr; } + bool operator!= ( const RandomIterator& r ) const { return my_ptr != r.my_ptr; } + difference_type operator- (const RandomIterator &r) const {return my_ptr - r.my_ptr;} + RandomIterator operator+ (difference_type n) const {return RandomIterator(my_ptr + n);} + bool operator< (const RandomIterator &r) const {return my_ptr < r.my_ptr;} +}; + +template +class ConstRandomIterator { + const T * my_ptr; +public: + typedef std::random_access_iterator_tag iterator_category; + typedef const T value_type; + typedef typename std::allocator::const_pointer pointer; + typedef typename std::allocator::const_reference reference; + typedef typename std::allocator::difference_type difference_type; + + explicit ConstRandomIterator ( const T * ptr ) : my_ptr(ptr){} + ConstRandomIterator ( const ConstRandomIterator& r ) : my_ptr(r.my_ptr){} + const T& operator* () const { return *my_ptr; } + ConstRandomIterator& operator++ () { ++my_ptr; return *this; } + bool operator== ( const ConstRandomIterator& r ) const { return my_ptr == r.my_ptr; } + bool operator!= ( const ConstRandomIterator& r ) const { return my_ptr != r.my_ptr; } + difference_type operator- (const ConstRandomIterator &r) const {return my_ptr - r.my_ptr;} + ConstRandomIterator operator+ (difference_type n) const {return ConstRandomIterator(my_ptr + n);} + bool operator< (const ConstRandomIterator &r) const {return my_ptr < r.my_ptr;} +}; + +} // namespace Harness + +#endif //harness_iterator_H diff --git a/src/tbb/src/test/harness_m128.h b/src/tbb/src/test/harness_m128.h new file mode 100644 index 00000000..3e4985b9 --- /dev/null +++ b/src/tbb/src/test/harness_m128.h @@ -0,0 +1,120 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Header that sets HAVE_m128/HAVE_m256 if vector types (__m128/__m256) are available + +//! Class for testing safety of using vector types. +/** Uses circuitous logic forces compiler to put __m128/__m256 objects on stack while + executing various methods, and thus tempt it to use aligned loads and stores + on the stack. */ +// Do not create file-scope objects of the class, because MinGW (as of May 2010) +// did not always provide proper stack alignment in destructors of such objects. + +#if (_MSC_VER>=1600) +//TODO: handle /arch:AVX in the right way. +#pragma warning (push) +#pragma warning (disable: 4752) +#endif + +#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT && __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif + + +template +class ClassWithVectorType { + static const int n = 16; + static const int F = sizeof(__Mvec)/sizeof(float); + __Mvec field[n]; + void init( int start ); +public: + ClassWithVectorType() {init(-n);} + ClassWithVectorType( int i ) {init(i);} + void operator=( const ClassWithVectorType& src ) { + __Mvec stack[n]; + for( int i=0; i +void ClassWithVectorType<__Mvec>::init( int start ) { + __Mvec stack[n]; + for( int i=0; i=1600 && _M_X64)) && !defined(__sun) +#include +#define HAVE_m256 1 +typedef ClassWithVectorType<__m256> ClassWithAVX; +#if _MSC_VER +#include // for __cpuid +#endif +bool have_AVX() { + bool result = false; + const int avx_mask = 1<<28; +#if _MSC_VER || __INTEL_COMPILER + int info[4] = {0,0,0,0}; + const int ECX = 2; + __cpuid(info, 1); + result = (info[ECX] & avx_mask)!=0; +#elif __GNUC__ + int ECX; + __asm__( "cpuid" + : "=c"(ECX) + : "a" (1) + : "ebx", "edx" ); + result = (ECX & avx_mask); +#endif + return result; +} +#endif /* __AVX__ etc */ + +#if (__SSE__ || _M_IX86_FP || _M_X64) && !defined(__sun) +#include +#define HAVE_m128 1 +typedef ClassWithVectorType<__m128> ClassWithSSE; +#endif + +#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT && __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT +#pragma GCC diagnostic pop +#endif + +#if (_MSC_VER>=1600) +#pragma warning (pop) +#endif diff --git a/src/tbb/src/test/harness_memory.h b/src/tbb/src/test/harness_memory.h new file mode 100644 index 00000000..0446fa3f --- /dev/null +++ b/src/tbb/src/test/harness_memory.h @@ -0,0 +1,141 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for simple estimate of the memory being used by a program. +// Not yet implemented for macOS*. +// This header is an optional part of the test harness. +// It assumes that "harness_assert.h" has already been included. + +#if __linux__ || __sun +#include +#include + +#elif __APPLE__ && !__ARM_ARCH +#include +#include +#include +#if MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_6 || __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0 +#include +#else +#include +#endif +#if SHARED_TEXT_REGION_SIZE || SHARED_DATA_REGION_SIZE +const size_t shared_size = SHARED_TEXT_REGION_SIZE+SHARED_DATA_REGION_SIZE; +#else +const size_t shared_size = 0; +#endif + +#elif _WIN32 && !__TBB_WIN8UI_SUPPORT +#include +#include +#if _MSC_VER +#pragma comment(lib, "psapi") +#endif + +#endif /* OS selection */ + +enum MemoryStatType { + currentUsage, + peakUsage +}; + +//! Return estimate of number of bytes of memory that this program is currently using. +/* Returns 0 if not implemented on platform. */ +size_t GetMemoryUsage(MemoryStatType stat = currentUsage) { + ASSERT(stat==currentUsage || stat==peakUsage, NULL); +#if __TBB_WIN8UI_SUPPORT + return 0; +#elif _WIN32 + PROCESS_MEMORY_COUNTERS mem; + bool status = GetProcessMemoryInfo(GetCurrentProcess(), &mem, sizeof(mem))!=0; + ASSERT(status, NULL); + return stat==currentUsage? mem.PagefileUsage : mem.PeakPagefileUsage; +#elif __linux__ + long unsigned size = 0; + FILE *fst = fopen("/proc/self/status", "r"); + ASSERT(fst, NULL); + const int BUF_SZ = 200; + char buf_stat[BUF_SZ]; + const char *pattern = stat==peakUsage ? "VmPeak: %lu" : "VmSize: %lu"; + while (NULL != fgets(buf_stat, BUF_SZ, fst)) { + if (1==sscanf(buf_stat, pattern, &size)) { + ASSERT(size, "Invalid value of memory consumption."); + break; + } + } + // VmPeak is available in kernels staring 2.6.15 + if (stat!=peakUsage || LinuxKernelVersion() >= 2006015) + ASSERT(size, "Invalid /proc/self/status format, pattern not found."); + fclose(fst); + return size*1024; +#elif __APPLE__ && !__ARM_ARCH + // TODO: find how detect peak virtual memory size under macOS + if (stat == peakUsage) + return 0; + kern_return_t status; + task_basic_info info; + mach_msg_type_number_t msg_type = TASK_BASIC_INFO_COUNT; + status = task_info(mach_task_self(), TASK_BASIC_INFO, reinterpret_cast(&info), &msg_type); + ASSERT(status==KERN_SUCCESS, NULL); + return info.virtual_size - shared_size; +#else + return 0; +#endif +} + +//! Use approximately a specified amount of stack space. +/** Recursion is used here instead of alloca because some implementations of alloca do not use the stack. */ +void UseStackSpace( size_t amount, char* top=0 ) { + char x[1000]; + memset( x, -1, sizeof(x) ); + if( !top ) + top = x; + ASSERT( x<=top, "test assumes that stacks grow downwards" ); + if( size_t(top-x)("/sys/kernel/mm/transparent_hugepage/enabled", thpItem); + + if (thpPresent == 'y') { + return true; + } else { + return false; + } +} +inline unsigned long long getSystemTHPAllocatedSize() { + unsigned long long anonHugePagesSize = 0; + parseFileItem meminfoItems[] = { + { "AnonHugePages: %llu kB", anonHugePagesSize } }; + parseFile("/proc/meminfo", meminfoItems); + return anonHugePagesSize; +} +inline unsigned long long getSystemTHPCount() { + unsigned long long anonHugePages = 0; + parseFileItem vmstatItems[] = { + { "nr_anon_transparent_hugepages %llu", anonHugePages } }; + parseFile("/proc/vmstat", vmstatItems); + return anonHugePages; +} +#endif // __linux__ + diff --git a/src/tbb/src/test/harness_mic.h b/src/tbb/src/test/harness_mic.h new file mode 100644 index 00000000..932bcc31 --- /dev/null +++ b/src/tbb/src/test/harness_mic.h @@ -0,0 +1,42 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_test_harness_mic_H +#define tbb_test_harness_mic_H + +#if ! __TBB_DEFINE_MIC + #error test/harness_mic.h should be included only when building for Intel(R) Many Integrated Core Architecture +#endif + +// test for unifed sources. See makefiles +#undef HARNESS_INCOMPLETE_SOURCES + +#include +#include + +#define TBB_TEST_LOW_WORKLOAD 1 + +#define REPORT_FATAL_ERROR REPORT +#define HARNESS_EXPORT + +#if __TBB_MIC_NATIVE + #define HARNESS_EXIT_ON_ASSERT 1 + #define __TBB_PLACEMENT_NEW_EXCEPTION_SAFETY_BROKEN 1 +#else + #define HARNESS_TERMINATE_ON_ASSERT 1 +#endif + +#endif /* tbb_test_harness_mic_H */ diff --git a/src/tbb/src/test/harness_preload.h b/src/tbb/src/test/harness_preload.h new file mode 100644 index 00000000..0851b262 --- /dev/null +++ b/src/tbb/src/test/harness_preload.h @@ -0,0 +1,43 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// This file is intended for preloading (via compiler options such as -include) into every test. +// Alas, not all compilers have such options, so the file is "optional". + +// Only add here things that are necessary for *every* test! +// In particular, avoid including other headers. +// Since this file can be omitted, checking compiler-specific conditions is strongly recommended. + +#ifndef harness_preload_H +#define harness_preload_H + +#if __GNUC__>=5 && !__INTEL_COMPILER && !__clang__ && __GXX_EXPERIMENTAL_CXX0X__ +// GCC 5 has added -Wsuggest-override, but unfortunately enables it even in pre-C++11 mode. +// We only want to use it for C++11 though. +#pragma GCC diagnostic warning "-Wsuggest-override" +#define __TBB_TEST_USE_WSUGGEST_OVERRIDE 1 +#endif +// TODO: consider adding a similar option for clang + +#if __TBB_TEST_NO_EXCEPTIONS +// This code breaks our own recommendations above, and it's deliberate: +// it includes another file, but that file should only have macros and pragmas; +// it does not check for compiler, as that is checked in the included file. +// The file also defines TBB_USE_EXCEPTIONS=0, which is set for all tests via makefiles anyway. +#include "tbb/tbb_disable_exceptions.h" +#endif + +#endif /* harness_preload_H */ diff --git a/src/tbb/src/test/harness_report.h b/src/tbb/src/test/harness_report.h new file mode 100644 index 00000000..86b80f8d --- /dev/null +++ b/src/tbb/src/test/harness_report.h @@ -0,0 +1,174 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Just the tracing portion of the harness. +// +// This header defines TRACE and TRACENL macros, which use REPORT like syntax and +// are useful for duplicating trace output to the standard debug output on Windows. +// It is possible to add the ability of automatic extending messages with additional +// info (file, line, function, time, thread ID, ...). +// +// Macros output nothing when test app runs in non-verbose mode (default). +// + +#ifndef tbb_tests_harness_report_H +#define tbb_tests_harness_report_H + +#if defined(MAX_TRACE_SIZE) && MAX_TRACE_SIZE < 1024 + #undef MAX_TRACE_SIZE +#endif +#ifndef MAX_TRACE_SIZE + #define MAX_TRACE_SIZE 1024 +#endif + +#if __SUNPRO_CC +#include +#else +#include +#endif + +#include + +// Need to include "tbb/tbb_config.h" to obtain the definition of __TBB_DEFINE_MIC. +#include "tbb/tbb_config.h" + +#if __TBB_DEFINE_MIC +#include "harness_mic.h" +#endif + +#ifdef HARNESS_INCOMPLETE_SOURCES +#error Source files are not complete. Check the build environment +#endif + +#if _MSC_VER + #define snprintf _snprintf +#if _MSC_VER<=1400 + #define vsnprintf _vsnprintf +#endif +#endif + +namespace Harness { + namespace internal { + +#ifndef TbbHarnessReporter + struct TbbHarnessReporter { + void Report ( const char* msg ) { + printf( "%s", msg ); + fflush(stdout); +#ifdef _WINDOWS_ + OutputDebugStringA(msg); +#endif + } + }; // struct TbbHarnessReporter +#endif /* !TbbHarnessReporter */ + + class Tracer { + int m_flags; + const char *m_file; + const char *m_func; + size_t m_line; + + TbbHarnessReporter m_reporter; + + public: + enum { + prefix = 1, + need_lf = 2 + }; + + Tracer(): m_flags(0), m_file(NULL), m_func(NULL), m_line(0) {} + + Tracer* set_trace_info ( int flags, const char *file, size_t line, const char *func ) { + m_flags = flags; + m_line = line; + m_file = file; + m_func = func; + return this; + } + + void trace ( const char* fmt, ... ) { + char msg[MAX_TRACE_SIZE]; + char msg_fmt_buf[MAX_TRACE_SIZE]; + const char *msg_fmt = fmt; + if ( m_flags & prefix ) { + snprintf (msg_fmt_buf, MAX_TRACE_SIZE, "[%s] %s", m_func, fmt); + msg_fmt = msg_fmt_buf; + } + std::va_list argptr; + va_start (argptr, fmt); + int len = vsnprintf (msg, MAX_TRACE_SIZE, msg_fmt, argptr); + va_end (argptr); + if ( m_flags & need_lf && + len < MAX_TRACE_SIZE - 1 && msg_fmt[len-1] != '\n' ) + { + msg[len] = '\n'; + msg[len + 1] = 0; + } + m_reporter.Report(msg); + } + }; // class Tracer + + static Tracer tracer; + + template + bool not_the_first_call () { + static bool first_call = false; + bool res = first_call; + first_call = true; + return res; + } + + } // namespace internal +} // namespace Harness + +#if defined(_MSC_VER) && _MSC_VER >= 1300 || defined(__GNUC__) || defined(__GNUG__) + #define HARNESS_TRACE_ORIG_INFO __FILE__, __LINE__, __FUNCTION__ +#else + #define HARNESS_TRACE_ORIG_INFO __FILE__, __LINE__, "" + #define __FUNCTION__ "" +#endif + + +//! printf style tracing macro +/** This variant of TRACE adds trailing line-feed (new line) character, if it is absent. **/ +#define TRACE Harness::internal::tracer.set_trace_info(Harness::internal::Tracer::need_lf, HARNESS_TRACE_ORIG_INFO)->trace + +//! printf style tracing macro without automatic new line character adding +#define TRACENL Harness::internal::tracer.set_trace_info(0, HARNESS_TRACE_ORIG_INFO)->trace + +//! printf style tracing macro with additional information prefix (e.g. current function name) +#define TRACEP Harness::internal::tracer.set_trace_info(Harness::internal::Tracer::prefix | \ + Harness::internal::Tracer::need_lf, HARNESS_TRACE_ORIG_INFO)->trace + +//! printf style remark macro +/** Produces output only when the test is run with the -v (verbose) option. **/ +#define REMARK !Verbose ? (void)0 : TRACENL + +//! printf style remark macro +/** Produces output only when invoked first time. + Only one instance of this macro is allowed per source code line. **/ +#define REMARK_ONCE (!Verbose || Harness::internal::not_the_first_call<__LINE__>()) ? (void)0 : TRACE + +//! printf style reporting macro +/** On heterogeneous platforms redirects its output to the host side. **/ +#define REPORT TRACENL + +//! printf style reporting macro +/** Produces output only when invoked first time. + Only one instance of this macro is allowed per source code line. **/ +#define REPORT_ONCE (Harness::internal::not_the_first_call<__LINE__>()) ? (void)0 : TRACENL + +#endif /* tbb_tests_harness_report_H */ diff --git a/src/tbb/src/test/harness_runtime_loader.h b/src/tbb/src/test/harness_runtime_loader.h new file mode 100644 index 00000000..516659be --- /dev/null +++ b/src/tbb/src/test/harness_runtime_loader.h @@ -0,0 +1,33 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef harness_runtime_loader_H +#define harness_runtime_loader_H + +#if HARNESS_USE_RUNTIME_LOADER + #if TEST_USES_TBB + #define TBB_PREVIEW_RUNTIME_LOADER 1 + #include "tbb/runtime_loader.h" + static char const * _path[] = { ".", NULL }; + // declaration must be placed before 1st TBB call + static tbb::runtime_loader _runtime_loader( _path ); + #else // TEST_USES_TBB + // if TBB library is not used, no need to test Runtime Loader + #define HARNESS_SKIP_TEST 1 + #endif // TEST_USES_TBB +#endif // HARNESS_USE_RUNTIME_LOADER + +#endif /* harness_runtime_loader_H */ diff --git a/src/tbb/src/test/harness_state_trackable.h b/src/tbb/src/test/harness_state_trackable.h new file mode 100644 index 00000000..437c7c9d --- /dev/null +++ b/src/tbb/src/test/harness_state_trackable.h @@ -0,0 +1,143 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Declarations for a class that can track operations applied to its objects. +// This header is an optional part of the test harness. + +#ifndef tbb_test_harness_state_trackable_H +#define tbb_test_harness_state_trackable_H + +#include +#include +#include + +#include "harness_assert.h" + +namespace Harness{ + struct StateTrackableBase { + enum StateValue { + ZeroInitialized = 0, + DefaultInitialized = 0xDEFAUL, + DirectInitialized = 0xD1111, + CopyInitialized = 0xC0314, + MoveInitialized = 0xAAAAA, + Assigned = 0x11AED, + MoveAssigned = 0x22AED, + MovedFrom = 0xFFFFF, + Destroyed = 0xDEADF00, + Unspecified = 0xEEEEE + }; + + class State { + public: + State() __TBB_NOEXCEPT(true) : state(Unspecified) { + assignNewState(Unspecified); + } + State(const State& s) : state(Unspecified) { + assignNewState(s.state); + } + State(StateValue s) __TBB_NOEXCEPT(true) : state(Unspecified) { + assignNewState(s); + }; + State& operator=(StateValue s) __TBB_NOEXCEPT(true) { + assignNewState(s); + return *this; + } + operator StateValue() const __TBB_NOEXCEPT(true) { return state; } + private: + void assignNewState(StateValue s) __TBB_NOEXCEPT(true); + StateValue state; + }; + }; + + struct StateTrackableCounters { + static void reset() { + counters[StateTrackableBase::ZeroInitialized] = counters[StateTrackableBase::DefaultInitialized] = + counters[StateTrackableBase::DirectInitialized] = counters[StateTrackableBase::CopyInitialized] = + counters[StateTrackableBase::MoveInitialized] = counters[StateTrackableBase::Assigned] = + counters[StateTrackableBase::MoveAssigned] = counters[StateTrackableBase::MovedFrom] = + counters[StateTrackableBase::Destroyed] = counters[StateTrackableBase::Unspecified] = 0; + } + + static bool initialize() { + reset(); + return true; + } + + typedef std::map > counters_t; + static counters_t counters; + }; + + StateTrackableCounters::counters_t StateTrackableCounters::counters; + static const bool stateTrackableBaseStateInitialized = StateTrackableCounters::initialize(); + + void StateTrackableBase::State::assignNewState(StateValue s) __TBB_NOEXCEPT(true) { + ASSERT(stateTrackableBaseStateInitialized, "State trackable counters are not initialized"); + ASSERT(s == StateTrackableBase::Unspecified || + StateTrackableCounters::counters.find(s) != StateTrackableCounters::counters.end(), "The current state value is unknown"); + ASSERT(state == StateTrackableBase::Unspecified || + StateTrackableCounters::counters.find(state) != StateTrackableCounters::counters.end(), "The new state value is unknown"); + state = s; + ++StateTrackableCounters::counters[state]; + } + + template + struct StateTrackable: StateTrackableBase { + static const bool is_zero_initialized_state_allowed = allow_zero_initialized_state; + State state; + + bool is_valid() const { + return state == DefaultInitialized || state == DirectInitialized || state == CopyInitialized + || state == MoveInitialized || state == Assigned || state == MoveAssigned || state == MovedFrom + || (allow_zero_initialized_state && state == ZeroInitialized) + ; + } + + StateTrackable (intptr_t) __TBB_NOEXCEPT(true) : state (DirectInitialized){} + StateTrackable () __TBB_NOEXCEPT(true) : state (DefaultInitialized){} + StateTrackable (const StateTrackable & src) __TBB_NOEXCEPT(true) { + ASSERT( src.is_valid(), "bad source for copy" ); + state = CopyInitialized; + } + #if __TBB_CPP11_RVALUE_REF_PRESENT + StateTrackable (StateTrackable && src) __TBB_NOEXCEPT(true) { + ASSERT( src.is_valid(), "bad source for move?" ); + state = MoveInitialized; + src.state = MovedFrom; + } + StateTrackable & operator=(StateTrackable && src) __TBB_NOEXCEPT(true) { + ASSERT( src.is_valid(), "bad source for assignment" ); + ASSERT( is_valid(), "assigning to invalid instance?" ); + + src.state = MovedFrom; + state = MoveAssigned; + return *this; + } + #endif + StateTrackable & operator=(const StateTrackable & src) __TBB_NOEXCEPT(true) { + ASSERT( src.is_valid(), "bad source for assignment?" ); + ASSERT( is_valid(), "assigning to invalid instance?" ); + + state = Assigned; + return *this; + } + ~StateTrackable () __TBB_NOEXCEPT(true) { + ASSERT( is_valid(), "Calling destructor on invalid instance? (twice destructor call?)" ); + state = Destroyed; + } + }; +} // Harness +#endif // tbb_test_harness_state_trackable_H diff --git a/src/tbb/src/test/harness_task.h b/src/tbb/src/test/harness_task.h new file mode 100644 index 00000000..7c5ec626 --- /dev/null +++ b/src/tbb/src/test/harness_task.h @@ -0,0 +1,51 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/task.h" +#include "harness.h" + +//! Helper for verifying that old use cases of spawn syntax still work. +tbb::task* GetTaskPtr( int& counter ) { + ++counter; + return NULL; +} + +class TaskGenerator: public tbb::task { + int m_ChildCount; + int m_Depth; + +public: + TaskGenerator( int child_count, int _depth ) : m_ChildCount(child_count), m_Depth(_depth) {} + ~TaskGenerator( ) { m_ChildCount = m_Depth = -125; } + + tbb::task* execute() __TBB_override { + ASSERT( m_ChildCount>=0 && m_Depth>=0, NULL ); + if( m_Depth>0 ) { + recycle_as_safe_continuation(); + set_ref_count( m_ChildCount+1 ); + int k=0; + for( int j=0; jspawn(t); + } + ASSERT(k==m_ChildCount,NULL); + --m_Depth; + __TBB_Yield(); + ASSERT( state()==recycle && ref_count()>0, NULL); + } + return NULL; + } +}; diff --git a/src/tbb/src/test/harness_tbb_independence.h b/src/tbb/src/test/harness_tbb_independence.h new file mode 100644 index 00000000..905a30d6 --- /dev/null +++ b/src/tbb/src/test/harness_tbb_independence.h @@ -0,0 +1,83 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef harness_tbb_independence_H +#define harness_tbb_independence_H + +// The tests which include tbb/atomic.h gain the dependency on the __TBB_ASSERT +// implementation even the test does not use anything from it. But almost all +// compilers optimize out unused inline function so they throw out the +// dependency. But to be pedantic with the standard the __TBB_ASSERT +// implementation should be provided. Moreover the offload compiler really +// requires it. +#include "../tbb/tbb_assert_impl.h" + +#if __linux__ && __ia64__ + +#define __TBB_NO_IMPLICIT_LINKAGE 1 +#include "tbb/tbb_machine.h" + +#include + +// Can't use Intel compiler intrinsic due to internal error reported by 10.1 compiler +pthread_mutex_t counter_mutex = PTHREAD_MUTEX_INITIALIZER; + +int32_t __TBB_machine_fetchadd4__TBB_full_fence (volatile void *ptr, int32_t value) +{ + pthread_mutex_lock(&counter_mutex); + int32_t result = *(int32_t*)ptr; + *(int32_t*)ptr = result + value; + pthread_mutex_unlock(&counter_mutex); + return result; +} + +int64_t __TBB_machine_fetchadd8__TBB_full_fence (volatile void *ptr, int64_t value) +{ + pthread_mutex_lock(&counter_mutex); + int32_t result = *(int32_t*)ptr; + *(int32_t*)ptr = result + value; + pthread_mutex_unlock(&counter_mutex); + return result; +} + +void __TBB_machine_pause(int32_t /*delay*/) { __TBB_Yield(); } + +pthread_mutex_t cas_mutex = PTHREAD_MUTEX_INITIALIZER; + +extern "C" int64_t __TBB_machine_cmpswp8__TBB_full_fence(volatile void *ptr, int64_t value, int64_t comparand) +{ + pthread_mutex_lock(&cas_mutex); + int64_t result = *(int64_t*)ptr; + if (result == comparand) + *(int64_t*)ptr = value; + pthread_mutex_unlock(&cas_mutex); + return result; +} + +pthread_mutex_t fetchstore_mutex = PTHREAD_MUTEX_INITIALIZER; + +int64_t __TBB_machine_fetchstore8__TBB_full_fence (volatile void *ptr, int64_t value) +{ + pthread_mutex_lock(&fetchstore_mutex); + int64_t result = *(int64_t*)ptr; + *(int64_t*)ptr = value; + pthread_mutex_unlock(&fetchstore_mutex); + return result; +} + +#endif /* __linux__ && __ia64 */ + +#endif // harness_tbb_independence_H diff --git a/src/tbb/src/test/harness_test_cases_framework.h b/src/tbb/src/test/harness_test_cases_framework.h new file mode 100644 index 00000000..694ed26e --- /dev/null +++ b/src/tbb/src/test/harness_test_cases_framework.h @@ -0,0 +1,236 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef tbb_harness_test_cases_framework_H +#define tbb_harness_test_cases_framework_H + +#if defined(_MSC_VER) + #define _SCL_SECURE_NO_WARNINGS +#endif + +#undef DO_ITT_NOTIFY + +#include "harness.h" +#include "harness_assert.h" +#include "tbb/tbb_stddef.h" + +#include + +#include +#include +#include +#include +#include + +namespace test_framework{ + template + void run_test(){ + test_class()(); + } + +#if TBB_USE_EXCEPTIONS + struct assertion_failure:std::exception{ + const char* my_filename; + int my_line; + const char* my_expression; + const char * my_comment; + assertion_failure(const char* filename, int line, const char* expression, const char * comment): + my_filename(filename), + my_line(line), + my_expression(expression), + my_comment(comment) + {} + virtual const char* what() const throw() __TBB_override { + return "test assertion failed"; + } + }; + void throw_assertion_failure(){throw assertion_failure("",0,"","");} + void throw_assertion_failure(const char* filename, int line, const char* expression, const char * comment){ + throw assertion_failure(filename, line, expression, comment); + } +#endif // TBB_USE_EXCEPTIONS + + class test_suite{ + typedef void(*run_test_function_pointer_type)(); + typedef std::pair tc_record_pair; + std::vector test_cases; + public: + template + void register_test_case(std::string const& name, test_class * ){ + test_cases.push_back(tc_record_pair(name,& run_test)); + } + std::string operator()(bool silent=false){ + std::stringstream str; + size_t failed=0; + for (size_t i=0;i(0)),true);\ + void TC_NAME::operator()() + +namespace test_framework_unit_tests{ + namespace test_helper{ + template struct tag{}; + template + struct test_case{ + static bool is_run; + void operator()(){ + is_run=true; + } + }; + template bool test_case::is_run = false; + + } + using namespace test_framework; + namespace test_test_suite_ref{ + void run_all_runs_all_registered_test_cases(){ + test_suite s; + using test_helper::tag; + test_helper::test_case > tc1; + test_helper::test_case > tc2; + s.register_test_case("tc1",&tc1); + s.register_test_case("tc2",&tc2); + s(); + ASSERT(tc1.is_run && tc2.is_run,"test_suite::operator() should run all the tests"); + } + + struct silent_switch_fixture{ + test_helper::test_case > empty_test_case; + }; + struct run_all_and_print_results_should_respect_silent_mode: silent_switch_fixture{ + void operator()(){ + using test_helper::tag; + test_helper::test_case > do_nothing_tc; + test_suite ts; + ts.register_test_case("tc_name",&do_nothing_tc); + bool silent =true; + ASSERT(ts(silent).empty(),"in silent mode no message except error should be output"); + } + }; + struct run_all_and_print_results_should_respect_verbose_mode: silent_switch_fixture{ + void operator()(){ + using test_helper::tag; + test_helper::test_case > do_nothing_tc; + test_suite ts; + ts.register_test_case("tc_name",&do_nothing_tc); + bool silent =true; + ASSERT(!ts(!silent).empty(),"in verbose mode all messages should be outputted"); + } + }; + } + namespace test_test_case_macro{ + test_suite& get_suite_ref(){static test_suite ts; return ts;} + typedef test_helper::test_case > unique_test_type; + TEST_CASE_WITH_FIXTURE(test_auto_registration,unique_test_type){ + unique_test_type::operator()(); + } + void run_test_test_case_macro(){ + get_suite_ref()(); + ASSERT(unique_test_type::is_run,"test case macro should register the test case in suite"); + } + void test_test_case_macro_does_not_create_test_case_object(){ + ASSERT(false,"to implement"); + } + } + namespace internal_assertions_failure_test_cases{ + + test_suite& get_suite_ref(){static test_suite ts; return ts;} + + //TODO: investigate compilation errors regarding tbb::set_assertion_handler +// struct empty_fixture{}; +// TEST_CASE_WITH_FIXTURE(test_internal_assertion_does_not_stop_test_suite,empty_fixture){ +// struct handler{ +// static void _( const char* /*filename*/, int /*line*/, const char* /*expression*/, const char * /*comment*/ ){ +// } +// }; +// +// tbb::assertion_handler_type previous = tbb::set_assertion_handler(handler::_); +// __TBB_ASSERT(false,"this assert should not stop the test suite run"); +// tbb::set_assertion_handler(previous ); +//// ASSERT(assertion_handler::is_called,"__TBB_ASSERT should call installed assertion handler"); +// } +// TEST_CASE_WITH_FIXTURE(test_internal_assertion_does_mark_the_test_as_failed,empty_fixture){ +// test_suite ts; +// struct _{ +//// static +// static void assertion_handler_type( const char* /*filename*/, int /*line*/, const char* /*expression*/, const char * /*comment*/ ){ +// } +// }; +// tbb::assertion_handler_type previous = tbb::set_assertion_handler(_::assertion_handler_type); +// __TBB_ASSERT(false,"this assert should not stop the test suite run"); +// tbb::set_assertion_handler(previous ); +// std::string result = ts(); +// std::size_t test_case_name_begin_pos = result.find("test case \""); +// std::size_t failed_begin_pos = result.find("failed"); +// ASSERT(test_case_name_begin_pos!=std::string::npos && failed_begin_pos!=std::string::npos && test_case_name_begin_pos= keep_keys-1, "Less TLS keys are available than requested"); + for (; keep_keys>0; keep_keys--, lastUsedIdx--) { +#if _WIN32 || _WIN64 + TlsFree(handles[lastUsedIdx]); +#else + int ret = pthread_key_delete(handles[lastUsedIdx]); + ASSERT(!ret, "Can't delete a key"); +#endif + } + REMARK("%d thread local objects allocated in advance\n", lastUsedIdx+1); + } + ~LimitTLSKeysTo() { + for (int i=0; i<=lastUsedIdx; i++) { +#if _WIN32 || _WIN64 + TlsFree(handles[i]); +#else + int ret = pthread_key_delete(handles[i]); + ASSERT(!ret, "Can't delete a key"); +#endif + } + lastUsedIdx = 0; + } +}; diff --git a/src/tbb/src/test/harness_tsx.h b/src/tbb/src/test/harness_tsx.h new file mode 100644 index 00000000..ecb4e955 --- /dev/null +++ b/src/tbb/src/test/harness_tsx.h @@ -0,0 +1,66 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Header that includes Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) specific test functions + +#if __TBB_TSX_AVAILABLE +#define __TBB_TSX_TESTING_ENABLED_FOR_THIS_COMPILER (__INTEL_COMPILER || __GNUC__ || _MSC_VER || __SUNPRO_CC) +#if __TBB_TSX_TESTING_ENABLED_FOR_THIS_COMPILER + +#include "harness_defs.h" + +inline static bool IsInsideTx() +{ + return __TBB_machine_is_in_transaction() != 0; +} + +#if _MSC_VER +#include // for __cpuid +#endif +// TODO: consider reusing tbb_misc.cpp:cpu_has_speculation() instead of code duplication. +bool have_TSX() { + bool result = false; + const int hle_ebx_mask = 1<<4; + const int rtm_ebx_mask = 1<<11; +#if _MSC_VER + int info[4] = {0,0,0,0}; + const int reg_ebx = 1; + int old_ecx = 0; + __cpuidex(info, 7, old_ecx); + result = (info[reg_ebx] & rtm_ebx_mask)!=0; + if( result ) ASSERT( (info[reg_ebx] & hle_ebx_mask)!=0, NULL ); +#elif __GNUC__ || __SUNPRO_CC + int32_t reg_ebx = 0; + int32_t reg_eax = 7; + int32_t reg_ecx = 0; + __asm__ __volatile__ ( "movl %%ebx, %%esi\n" + "cpuid\n" + "movl %%ebx, %0\n" + "movl %%esi, %%ebx\n" + : "=a"(reg_ebx) : "0" (reg_eax), "c" (reg_ecx) : "esi", +#if __TBB_x86_64 + "ebx", +#endif + "edx" + ); + result = (reg_ebx & rtm_ebx_mask)!=0 ; + if( result ) ASSERT( (reg_ebx & hle_ebx_mask)!=0, NULL ); +#endif + return result; +} + +#endif /* __TBB_TSX_TESTING_ENABLED_FOR_THIS_COMPILER */ +#endif /* __TBB_TSX_AVAILABLE */ diff --git a/src/tbb/src/test/test_ScalableAllocator.cpp b/src/tbb/src/test/test_ScalableAllocator.cpp new file mode 100644 index 00000000..b1604af0 --- /dev/null +++ b/src/tbb/src/test/test_ScalableAllocator.cpp @@ -0,0 +1,223 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Test whether scalable_allocator complies with the requirements in 20.1.5 of ISO C++ Standard (1998). + +#define __TBB_EXTRA_DEBUG 1 // enables additional checks +#define TBB_PREVIEW_MEMORY_POOL 1 + +#include "harness_assert.h" +#if !__TBB_SOURCE_DIRECTLY_INCLUDED +#include "harness_tbb_independence.h" // because harness_allocator.h requires atomics +#endif +#include "tbb/memory_pool.h" +#include "tbb/scalable_allocator.h" + +#define HARNESS_TBBMALLOC_THREAD_SHUTDOWN 1 +// the actual body of the test is there: +#include "test_allocator.h" +#include "harness_allocator.h" + +#if _MSC_VER +#include "tbb/machine/windows_api.h" +#endif /* _MSC_VER */ + +typedef static_counting_allocator > cnt_alloc_t; +typedef local_counting_allocator > cnt_provider_t; +class MinimalAllocator : cnt_provider_t { +public: + typedef char value_type; + MinimalAllocator() { + REMARK("%p::ctor\n", this); + } + MinimalAllocator(const MinimalAllocator&s) : cnt_provider_t(s) { + REMARK("%p::ctor(%p)\n", this, &s); + } + ~MinimalAllocator() { + REMARK("%p::dtor: alloc=%u/%u free=%u/%u\n", this, + unsigned(items_allocated),unsigned(allocations), + unsigned(items_freed), unsigned(frees) ); + ASSERT(allocations==frees && items_allocated==items_freed,0); + if( allocations ) { // non-temporal copy + // TODO: describe consumption requirements + ASSERT(items_allocated>cnt_alloc_t::items_allocated, 0); + } + } + void *allocate(size_t sz) { + void *p = cnt_provider_t::allocate(sz); + REMARK("%p::allocate(%u) = %p\n", this, unsigned(sz), p); + return p; + } + void deallocate(void *p, size_t sz) { + ASSERT(allocations>frees,0); + REMARK("%p::deallocate(%p, %u)\n", this, p, unsigned(sz)); + cnt_provider_t::deallocate(cnt_provider_t::pointer(p), sz); + } +}; + +class NullAllocator { +public: + typedef char value_type; + NullAllocator() { } + NullAllocator(const NullAllocator&) { } + ~NullAllocator() { } + void *allocate(size_t) { return NULL; } + void deallocate(void *, size_t) { ASSERT(0, NULL); } +}; + +void TestZeroSpaceMemoryPool() +{ + tbb::memory_pool pool; + bool allocated = pool.malloc(16) || pool.malloc(9*1024); + ASSERT(!allocated, "Allocator with no memory must not allocate anything."); +} + +#if !TBB_USE_EXCEPTIONS +struct FixedPool { + void *buf; + size_t size; + bool used; + FixedPool(void *a_buf, size_t a_size) : buf(a_buf), size(a_size), used(false) {} +}; + +static void *fixedBufGetMem(intptr_t pool_id, size_t &bytes) +{ + if (((FixedPool*)pool_id)->used) + return NULL; + + ((FixedPool*)pool_id)->used = true; + bytes = ((FixedPool*)pool_id)->size; + return bytes? ((FixedPool*)pool_id)->buf : NULL; +} +#endif + +/* test that pools in small space are either usable or not created + (i.e., exception raised) */ +void TestSmallFixedSizePool() +{ + char *buf; + bool allocated = false; + + for (size_t sz = 0; sz < 64*1024; sz = sz? 3*sz : 3) { + buf = (char*)malloc(sz); +#if TBB_USE_EXCEPTIONS + try { + tbb::fixed_pool pool(buf, sz); +/* Check that pool is usable, i.e. such an allocation exists, + that can be fulfilled from the pool. 16B allocation fits in 16KB slabs, + so it requires at least 16KB. Requirement of 9KB allocation is more modest. +*/ + allocated = pool.malloc( 16 ) || pool.malloc( 9*1024 ); + } catch (std::invalid_argument&) { + ASSERT(!sz, "expect std::invalid_argument for zero-sized pool only"); + } catch (...) { + ASSERT(0, "wrong exception type;"); + } +#else +/* Do not test high-level pool interface because pool ctor emit exception + on creation failure. Instead test same functionality via low-level interface. + TODO: add support for configuration with disabled exceptions to pools. +*/ + rml::MemPoolPolicy pol(fixedBufGetMem, NULL, 0, /*fixedSizePool=*/true, + /*keepMemTillDestroy=*/false); + rml::MemoryPool *pool; + FixedPool fixedPool(buf, sz); + + rml::MemPoolError ret = pool_create_v1((intptr_t)&fixedPool, &pol, &pool); + + if (ret == rml::POOL_OK) { + allocated = pool_malloc(pool, 16) || pool_malloc(pool, 9*1024); + pool_destroy(pool); + } else + ASSERT(ret == rml::NO_MEMORY, "Expected that pool either valid " + "or have no memory to be created"); +#endif + free(buf); + } + ASSERT(allocated, "Maximal buf size should be enough to create working fixed_pool"); +#if TBB_USE_EXCEPTIONS + try { + tbb::fixed_pool pool(NULL, 10*1024*1024); + ASSERT(0, "Useless allocator with no memory must not be created"); + } catch (std::invalid_argument&) { + } catch (...) { + ASSERT(0, "wrong exception type; expected invalid_argument"); + } +#endif +} + +int TestMain () { +#if _MSC_VER && !__TBBMALLOC_NO_IMPLICIT_LINKAGE && !__TBB_WIN8UI_SUPPORT + #ifdef _DEBUG + ASSERT(!GetModuleHandle("tbbmalloc.dll") && GetModuleHandle("tbbmalloc_debug.dll"), + "test linked with wrong (non-debug) tbbmalloc library"); + #else + ASSERT(!GetModuleHandle("tbbmalloc_debug.dll") && GetModuleHandle("tbbmalloc.dll"), + "test linked with wrong (debug) tbbmalloc library"); + #endif +#endif /* _MSC_VER && !__TBBMALLOC_NO_IMPLICIT_LINKAGE */ + int result = TestMain >(); + { + tbb::memory_pool > pool; + result += TestMain(tbb::memory_pool_allocator(pool) ); + }{ + tbb::memory_pool pool; + cnt_alloc_t alloc(( tbb::memory_pool_allocator(pool) )); // double parentheses to avoid function declaration + result += TestMain(alloc); + }{ + static char buf[1024*1024*4]; + tbb::fixed_pool pool(buf, sizeof(buf)); + const char *text = "this is a test";// 15 bytes + char *p1 = (char*)pool.malloc( 16 ); + ASSERT(p1, NULL); + strcpy(p1, text); + char *p2 = (char*)pool.realloc( p1, 15 ); + ASSERT( p2 && !strcmp(p2, text), "realloc broke memory" ); + + result += TestMain(tbb::memory_pool_allocator(pool) ); + + // try allocate almost entire buf keeping some reasonable space for internals + char *p3 = (char*)pool.realloc( p2, sizeof(buf)-128*1024 ); + ASSERT( p3, "defragmentation failed" ); + ASSERT( !strcmp(p3, text), "realloc broke memory" ); + for( size_t sz = 10; sz < sizeof(buf); sz *= 2) { + ASSERT( pool.malloc( sz ), NULL); + pool.recycle(); + } + + result += TestMain(tbb::memory_pool_allocator(pool) ); + }{ + // Two nested level allocators case with fixed pool allocator as an underlying layer + // serving allocRawMem requests for the top level scalable allocator + typedef tbb::memory_pool > NestedPool; + + static char buffer[8*1024*1024]; + tbb::fixed_pool fixedPool(buffer, sizeof(buffer)); + // Underlying fixed pool allocator + tbb::memory_pool_allocator fixedPoolAllocator(fixedPool); + // Memory pool that handles fixed pool allocator + NestedPool nestedPool(fixedPoolAllocator); + // Top level memory pool allocator + tbb::memory_pool_allocator nestedAllocator(nestedPool); + + result += TestMain(nestedAllocator); + } + TestSmallFixedSizePool(); + TestZeroSpaceMemoryPool(); + + ASSERT( !result, NULL ); + return Harness::Done; +} diff --git a/src/tbb/src/test/test_ScalableAllocator_STL.cpp b/src/tbb/src/test/test_ScalableAllocator_STL.cpp new file mode 100644 index 00000000..3989e966 --- /dev/null +++ b/src/tbb/src/test/test_ScalableAllocator_STL.cpp @@ -0,0 +1,50 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Test whether scalable_allocator works with some of the host's STL containers. + +#define HARNESS_NO_PARSE_COMMAND_LINE 1 +#define __TBB_EXTRA_DEBUG 1 // enables additional checks +#define TBB_PREVIEW_MEMORY_POOL 1 + +#include "harness_assert.h" +#include "tbb/memory_pool.h" +#include "tbb/scalable_allocator.h" +#include + +// The actual body of the test is there: +#include "test_allocator_STL.h" + +int TestMain () { + TestAllocatorWithSTL >(); + tbb::memory_pool > mpool; + TestAllocatorWithSTL(tbb::memory_pool_allocator(mpool) ); + static char buf[1024*1024*4]; + tbb::fixed_pool fpool(buf, sizeof(buf)); + TestAllocatorWithSTL(tbb::memory_pool_allocator(fpool) ); + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + ASSERT(!tbb::scalable_memory_resource()->is_equal(*std::pmr::get_default_resource()), + "Scalable resource shouldn't be equal to standard resource." ); + ASSERT(tbb::scalable_memory_resource()->is_equal(*tbb::scalable_memory_resource()), + "Memory that was allocated by one scalable resource should be deallocated by any other instance."); + + typedef std::pmr::polymorphic_allocator pmr_alloc_t; + TestAllocatorWithSTL(pmr_alloc_t(tbb::scalable_memory_resource())); +#endif + + return Harness::Done; +} diff --git a/src/tbb/src/test/test_aggregator.cpp b/src/tbb/src/test/test_aggregator.cpp new file mode 100644 index 00000000..601ab4ff --- /dev/null +++ b/src/tbb/src/test/test_aggregator.cpp @@ -0,0 +1,181 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef TBB_PREVIEW_AGGREGATOR + #define TBB_PREVIEW_AGGREGATOR 1 +#endif + +#include "tbb/aggregator.h" +#include "harness.h" +#include + +typedef std::priority_queue, std::less > pq_t; + +int N; +int* shared_data; + +// Code for testing basic interface using function objects +class push_fnobj : NoAssign, Harness::NoAfterlife { + pq_t& pq; + int threadID; +public: + push_fnobj(pq_t& pq_, int tid) : pq(pq_), threadID(tid) {} + void operator()() const { + AssertLive(); + pq.push(threadID); + } +}; + +class pop_fnobj : NoAssign, Harness::NoAfterlife { + pq_t& pq; +public: + pop_fnobj(pq_t& pq_) : pq(pq_) {} + void operator()() const { + AssertLive(); + ASSERT(!pq.empty(), "queue should not be empty yet"); + int elem = pq.top(); + pq.pop(); + shared_data[elem]++; + } +}; + +class BasicBody : NoAssign { + pq_t& pq; + tbb::aggregator& agg; +public: + BasicBody(pq_t& pq_, tbb::aggregator& agg_) : pq(pq_), agg(agg_) {} + void operator()(const int threadID) const { + for (int i=0; i(*op_list); + op_list = op_list->next(); + request.start(); + if (request.tid >= 0) pq->push(request.tid); + else { + ASSERT(!pq->empty(), "queue should not be empty!"); + int elem = pq->top(); + pq->pop(); + shared_data[elem]++; + } + request.finish(); + } + } +}; + +class ExpertBody : NoAssign { + pq_t& pq; + tbb::aggregator_ext& agg; +public: + ExpertBody(pq_t& pq_, tbb::aggregator_ext& agg_) : pq(pq_), agg(agg_) {} + void operator()(const int threadID) const { + for (int i=0; i agg((my_handler(&my_pq))); + for (int i=0; i never calls member of T. */ +template +class Minimal { + Minimal(); + Minimal( Minimal& min ); + ~Minimal(); + void operator=( const Minimal& ); + T pad; + template + friend void AssignToCheckAlignment( Minimal& dst, const Minimal& src ) ; +}; + +template +void AssignToCheckAlignment( Minimal& dst, const Minimal& src ) { + dst.pad = src.pad; +} + +#include "tbb/aligned_space.h" +#include "harness_assert.h" + +static bool SpaceWasted; + +template +void TestAlignedSpaceN() { + typedef Minimal T; + struct { + //! Pad byte increases chance that subsequent member will be misaligned if there is a problem. + char pad; + tbb::aligned_space space; + } x; + AssertSameType( static_cast< T *>(0), x.space.begin() ); + AssertSameType( static_cast< T *>(0), x.space.end() ); + ASSERT( reinterpret_cast(x.space.begin())==reinterpret_cast< void *>(&x.space), NULL ); + ASSERT( x.space.end()-x.space.begin()==N, NULL ); + ASSERT( reinterpret_cast(x.space.begin())>=reinterpret_cast< void *>(&x.space), NULL ); + ASSERT( x.space.end()<=reinterpret_cast< T *>(&x.space+1), NULL ); + // Though not required, a good implementation of aligned_space does not use any more space than a T[N]. + SpaceWasted |= sizeof(x.space)!=sizeof(T)*N; + for( size_t k=1; k +template +void PrintSpaceWastingWarning() { + REPORT( "Consider rewriting aligned_space<%s,N> to waste less space\n", typeid(T).name() ); +} + +// RTTI for long double (128 bit) is broken in libc++ up-to NDK11c. Check on newer versions of NDK. +#if ( __ANDROID__ && __clang__ && _LIBCPP_VERSION && __TBB_x86_64 ) +template<> +void PrintSpaceWastingWarning() { + REPORT( "Consider rewriting aligned_space to waste less space\n" ); +} +#endif + +template +void TestAlignedSpace() { + SpaceWasted = false; + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + TestAlignedSpaceN(); + if( SpaceWasted ) + PrintSpaceWastingWarning(); +} + +#include "harness_m128.h" + +int TestMain () { + TestAlignedSpace(); + TestAlignedSpace(); + TestAlignedSpace(); + TestAlignedSpace(); + TestAlignedSpace(); + TestAlignedSpace(); + TestAlignedSpace(); +#if HAVE_m128 + TestAlignedSpace<__m128>(); +#endif +#if HAVE_m256 + if (have_AVX()) TestAlignedSpace<__m256>(); +#endif + return Harness::Done; +} diff --git a/src/tbb/src/test/test_allocator.h b/src/tbb/src/test/test_allocator.h new file mode 100644 index 00000000..96e9dad5 --- /dev/null +++ b/src/tbb/src/test/test_allocator.h @@ -0,0 +1,271 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Basic testing of an allocator +// Tests against requirements in 20.1.5 of ISO C++ Standard (1998). +// Does not check for thread safety or false sharing issues. +// +// Tests for compatibility with the host's STL are in +// test_Allocator_STL.h. Those tests are in a separate file +// because they bring in lots of STL headers, and the tests here +// are supposed to work in the abscense of STL. + +#include "harness.h" +#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + #include //for std::pair +#endif + +template +struct is_zero_filling { + static const bool value = false; +}; + +int NumberOfFoo; + +template +struct Foo { + T foo_array[N]; + Foo() { + zero_fill(foo_array, N); + ++NumberOfFoo; + } + Foo( const Foo& x ) { + *this = x; + ++NumberOfFoo; + } + ~Foo() { + --NumberOfFoo; + } +}; + +inline char PseudoRandomValue( size_t j, size_t k ) { + return char(j*3 ^ j>>4 ^ k); +} + +#if __APPLE__ +#include +#include + +// A RAII class to disable stderr in a certain scope. It's not thread-safe. +class DisableStderr { + int stderrCopy; + static void dupToStderrAndClose(int fd) { + int ret = dup2(fd, STDERR_FILENO); // close current stderr + ASSERT(ret != -1, NULL); + ret = close(fd); + ASSERT(ret != -1, NULL); + } +public: + DisableStderr() { + int devNull = open("/dev/null", O_WRONLY); + ASSERT(devNull != -1, NULL); + stderrCopy = dup(STDERR_FILENO); + ASSERT(stderrCopy != -1, NULL); + dupToStderrAndClose(devNull); + } + ~DisableStderr() { + dupToStderrAndClose(stderrCopy); + } +}; +#endif + +//! T is type and A is allocator for that type +template +void TestBasic( A& a ) { + T x; + const T cx = T(); + + // See Table 32 in ISO ++ Standard + typename A::pointer px = &x; + typename A::const_pointer pcx = &cx; + + typename A::reference rx = x; + ASSERT( &rx==&x, NULL ); + + typename A::const_reference rcx = cx; + ASSERT( &rcx==&cx, NULL ); + + typename A::value_type v = x; + + typename A::size_type size; + size = 0; + --size; + ASSERT( size>0, "not an unsigned integral type?" ); + + typename A::difference_type difference; + difference = 0; + --difference; + ASSERT( difference<0, "not an signed integral type?" ); + + // "rebind" tested by our caller + + ASSERT( a.address(rx)==px, NULL ); + + ASSERT( a.address(rcx)==pcx, NULL ); + + typename A::pointer array[100]; + size_t sizeof_T = sizeof(T); + for( size_t k=0; k<100; ++k ) { + array[k] = k&1 ? a.allocate(k,array[0]) : a.allocate(k); + char* s = reinterpret_cast(reinterpret_cast(array[k])); + for( size_t j=0; j(reinterpret_cast(array[k])); + for( size_t j=0; j=a.max_size(), "max_size larger than reasonable" ); + + // Test "a.construct(p,t)" + int n = NumberOfFoo; + typename A::pointer p = a.allocate(1); + a.construct( p, cx ); + ASSERT( NumberOfFoo==n+1, "constructor for Foo not called?" ); + + // Test "a.destroy(p)" + a.destroy( p ); + ASSERT( NumberOfFoo==n, "destructor for Foo not called?" ); + a.deallocate(p,1); + +#if TBB_USE_EXCEPTIONS + volatile size_t too_big = (~size_t(0) - 1024*1024)/sizeof(T); + bool exception_caught = false; + typename A::pointer p1 = NULL; + try { +#if __APPLE__ + // On macOS*, failure to map memory results in messages to stderr; + // suppress them. + DisableStderr disableStderr; +#endif + p1 = a.allocate(too_big); + } catch ( std::bad_alloc& ) { + exception_caught = true; + } + ASSERT( exception_caught, "allocate expected to throw bad_alloc" ); + a.deallocate(p1, too_big); +#endif // TBB_USE_EXCEPTIONS + + #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC + { + typedef typename A:: template rebind >::other pair_allocator_type; + pair_allocator_type pair_allocator(a); + int NumberOfFooBeforeConstruct= NumberOfFoo; + typename pair_allocator_type::pointer pair_pointer = pair_allocator.allocate(1); + pair_allocator.construct( pair_pointer, cx, cx); + ASSERT( NumberOfFoo==NumberOfFooBeforeConstruct+2, "constructor for Foo not called appropriate number of times?" ); + + pair_allocator.destroy( pair_pointer ); + ASSERT( NumberOfFoo==NumberOfFooBeforeConstruct, "destructor for Foo not called appropriate number of times?" ); + pair_allocator.deallocate(pair_pointer,1); + } + #endif + +} + +#include "tbb/blocked_range.h" + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Workaround for erroneous "conditional expression is constant" warning in method check_allocate. + #pragma warning (disable: 4127) +#endif + +// A is an allocator for some type +template +struct Body: NoAssign { + static const size_t max_k = 100000; + A &a; + Body(A &a_) : a(a_) {} + void check_allocate( typename A::pointer array[], size_t i, size_t t ) const + { + ASSERT(array[i] == 0, NULL); + size_t size = i * (i&3); + array[i] = i&1 ? a.allocate(size, array[i>>3]) : a.allocate(size); + ASSERT(array[i] != 0, "allocator returned null"); + char* s = reinterpret_cast(reinterpret_cast(array[i])); + for( size_t j=0; j::other>::value) + ASSERT( !s[j], NULL); + s[j] = PseudoRandomValue(i, t); + } + } + + void check_deallocate( typename A::pointer array[], size_t i, size_t t ) const + { + ASSERT(array[i] != 0, NULL); + size_t size = i * (i&3); + char* s = reinterpret_cast(reinterpret_cast(array[i])); + for( size_t j=0; j(PseudoRandomValue(k,thread_id)); + if(!array[i]) check_allocate(array, i, thread_id); + else check_deallocate(array, i, thread_id); + } + for( size_t k=0; k<256; ++k ) + if(array[k]) + check_deallocate(array, k, thread_id); + } +}; + +// A is an allocator for some type, and U is another type +template +void Test(A &a) { + typename A::template rebind::other b(a); + TestBasic(b); + TestBasic(a); + + // thread safety + NativeParallelFor( 4, Body(a) ); + ASSERT( NumberOfFoo==0, "Allocate/deallocate count mismatched" ); + + ASSERT( a==b, NULL ); + ASSERT( !(a!=b), NULL ); +} + +template +int TestMain(const Allocator &a = Allocator()) { + NumberOfFoo = 0; + typename Allocator::template rebind >::other a1(a); + typename Allocator::template rebind >::other a2(a); + Test >( a1 ); + Test >( a2 ); + return 0; +} diff --git a/src/tbb/src/test/test_allocator_STL.h b/src/tbb/src/test/test_allocator_STL.h new file mode 100644 index 00000000..ac01eefc --- /dev/null +++ b/src/tbb/src/test/test_allocator_STL.h @@ -0,0 +1,147 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Tests for compatibility with the host's STL. + +#include "harness.h" + +template +void TestSequence(const typename Container::allocator_type &a) { + Container c(a); + for( int i=0; i<1000; ++i ) + c.push_back(i*i); + typename Container::const_iterator p = c.begin(); + for( int i=0; i<1000; ++i ) { + ASSERT( *p==i*i, NULL ); + ++p; + } + // regression test against compilation error for GCC 4.6.2 + c.resize(1000); +} + +template +void TestSet(const typename Set::allocator_type &a) { + Set s(typename Set::key_compare(), a); + typedef typename Set::value_type value_type; + for( int i=0; i<100; ++i ) + s.insert(value_type(3*i)); + for( int i=0; i<300; ++i ) { + ASSERT( s.erase(i)==size_t(i%3==0), NULL ); + } +} + +template +void TestMap(const typename Map::allocator_type &a) { + Map m(typename Map::key_compare(), a); + typedef typename Map::value_type value_type; + for( int i=0; i<100; ++i ) + m.insert(value_type(i,i*i)); + for( int i=0; i<100; ++i ) + ASSERT( m.find(i)->second==i*i, NULL ); +} + +#include +#include +#include +#include +#include + +#if __TBB_CPP11_RVALUE_REF_PRESENT +struct MoveOperationTracker { + int my_value; + + MoveOperationTracker( int value = 0 ) : my_value( value ) {} + MoveOperationTracker(const MoveOperationTracker&) { + ASSERT( false, "Copy constructor is called" ); + } + MoveOperationTracker(MoveOperationTracker&& m) __TBB_NOEXCEPT( true ) : my_value( m.my_value ) { + } + MoveOperationTracker& operator=(MoveOperationTracker const&) { + ASSERT( false, "Copy assignment operator is called" ); + return *this; + } + MoveOperationTracker& operator=(MoveOperationTracker&& m) __TBB_NOEXCEPT( true ) { + my_value = m.my_value; + return *this; + } + + bool operator==(int value) const { + return my_value == value; + } + + bool operator==(const MoveOperationTracker& m) const { + return my_value == m.my_value; + } +}; +#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ + +template +void TestAllocatorWithSTL(const Allocator &a = Allocator() ) { + +// Allocator type conversion section +#if __TBB_ALLOCATOR_TRAITS_PRESENT + typedef typename std::allocator_traits::template rebind_alloc Ai; + typedef typename std::allocator_traits::template rebind_alloc > Acii; +#if _MSC_VER + typedef typename std::allocator_traits::template rebind_alloc Aci; + typedef typename std::allocator_traits::template rebind_alloc > Aii; +#endif // _MSC_VER +#else + typedef typename Allocator::template rebind::other Ai; + typedef typename Allocator::template rebind >::other Acii; +#if _MSC_VER + typedef typename Allocator::template rebind::other Aci; + typedef typename Allocator::template rebind >::other Aii; +#endif // _MSC_VER +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + + // Sequenced containers + TestSequence >(a); + TestSequence >(a); + TestSequence >(a); + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#if __TBB_ALLOCATOR_TRAITS_PRESENT + typedef typename std::allocator_traits::template rebind_alloc Amot; +#else + typedef typename Allocator::template rebind::other Amot; +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + TestSequence >(a); + TestSequence >(a); + TestSequence >(a); +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + + // Associative containers + TestSet, Ai> >(a); + TestSet, Ai> >(a); + TestMap, Acii> >(a); + TestMap, Acii> >(a); + +#if _MSC_VER && _CPPLIB_VER < 650 + // Test compatibility with Microsoft's implementation of std::allocator for some cases that + // are undefined according to the ISO standard but permitted by Microsoft. + TestSequence >(a); +#if _CPPLIB_VER>=500 + TestSequence >(a); +#endif + TestSequence >(a); + TestSet, Aci> >(a); + TestMap, Aii> >(a); + TestMap, Acii> >(a); + TestMap, Aii> >(a); + TestMap, Acii> >(a); +#endif /* _MSC_VER */ +} diff --git a/src/tbb/src/test/test_assembly.cpp b/src/tbb/src/test/test_assembly.cpp new file mode 100644 index 00000000..562f4c4e --- /dev/null +++ b/src/tbb/src/test/test_assembly.cpp @@ -0,0 +1,160 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Program for basic correctness testing of assembly-language routines. +#include "harness_defs.h" +//for ICC builtins mode the test will be skipped as +//macro __TBB_GCC_BUILTIN_ATOMICS_PRESENT used to define __TBB_TEST_SKIP_GCC_BUILTINS_MODE +//will not be defined (it is explicitly disabled for ICC) +#if __TBB_TEST_SKIP_GCC_BUILTINS_MODE +#include "harness.h" +int TestMain() { + REPORT("Known issue: GCC builtins aren't available\n"); + return Harness::Skipped; +} +#else + +#include "tbb/task.h" + +#include +#include "harness.h" + +using tbb::internal::reference_count; + +//TODO: remove this function when atomic function __TBB_XXX are dropped +//! Test __TBB_CompareAndSwapW +static void TestCompareExchange() { + ASSERT( intptr_t(-10)<10, "intptr_t not a signed integral type?" ); + REMARK("testing __TBB_CompareAndSwapW\n"); + for( intptr_t a=-10; a<10; ++a ) + for( intptr_t b=-10; b<10; ++b ) + for( intptr_t c=-10; c<10; ++c ) { +// Workaround for a bug in GCC 4.3.0; and one more is below. +#if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN + intptr_t x; + __TBB_store_with_release( x, a ); +#else + intptr_t x = a; +#endif + intptr_t y = __TBB_CompareAndSwapW(&x,b,c); + ASSERT( y==a, NULL ); + if( a==c ) + ASSERT( x==b, NULL ); + else + ASSERT( x==a, NULL ); + } +} + +//TODO: remove this function when atomic function __TBB_XXX are dropped +//! Test __TBB___TBB_FetchAndIncrement and __TBB___TBB_FetchAndDecrement +static void TestAtomicCounter() { + // "canary" is a value used to detect illegal overwrites. + const reference_count canary = ~(uintptr_t)0/3; + REMARK("testing __TBB_FetchAndIncrement\n"); + struct { + reference_count prefix, i, suffix; + } x; + x.prefix = canary; + x.i = 0; + x.suffix = canary; + for( int k=0; k<10; ++k ) { + reference_count j = __TBB_FetchAndIncrementWacquire((volatile void *)&x.i); + ASSERT( x.prefix==canary, NULL ); + ASSERT( x.suffix==canary, NULL ); + ASSERT( x.i==k+1, NULL ); + ASSERT( j==k, NULL ); + } + REMARK("testing __TBB_FetchAndDecrement\n"); + x.i = 10; + for( int k=10; k>0; --k ) { + reference_count j = __TBB_FetchAndDecrementWrelease((volatile void *)&x.i); + ASSERT( j==k, NULL ); + ASSERT( x.i==k-1, NULL ); + ASSERT( x.prefix==canary, NULL ); + ASSERT( x.suffix==canary, NULL ); + } +} + +static void TestTinyLock() { + REMARK("testing __TBB_LockByte\n"); + __TBB_atomic_flag flags[16]; + for( unsigned int i=0; i<16; ++i ) + flags[i] = (__TBB_Flag)i; +#if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN + __TBB_store_with_release( flags[8], 0 ); +#else + flags[8] = 0; +#endif + __TBB_LockByte(flags[8]); + for( unsigned int i=0; i<16; ++i ) + #ifdef __sparc + ASSERT( flags[i]==(i==8?0xff:i), NULL ); + #else + ASSERT( flags[i]==(i==8?1:i), NULL ); + #endif + __TBB_UnlockByte(flags[8]); + for( unsigned int i=0; i<16; ++i ) + ASSERT( flags[i] == (i==8?0:i), NULL ); +} + +static void TestLog2() { + REMARK("testing __TBB_Log2\n"); + for( uintptr_t i=1; i; i<<=1 ) { + for( uintptr_t j=1; j<1<<16; ++j ) { + if( uintptr_t k = i*j ) { + uintptr_t actual = __TBB_Log2(k); + const uintptr_t ONE = 1; // warning suppression again + ASSERT( k >= ONE<>1 < ONE<prev, "__TBB_time_stamp has returned non-monotonically increasing quantity"); + prev=curr; + } + REMARK("\n"); +#else + REMARK(" skipped\n"); +#endif +} + +int TestMain () { + __TBB_TRY { + TestLog2(); + TestTinyLock(); + TestCompareExchange(); + TestAtomicCounter(); + TestPause(); + TestTimeStamp(); + } __TBB_CATCH(...) { + ASSERT(0,"unexpected exception"); + } + return Harness::Done; +} +#endif // __TBB_TEST_SKIP_BUILTINS_MODE diff --git a/src/tbb/src/test/test_async_msg.cpp b/src/tbb/src/test/test_async_msg.cpp new file mode 100644 index 00000000..76d9d3be --- /dev/null +++ b/src/tbb/src/test/test_async_msg.cpp @@ -0,0 +1,604 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef TBB_PREVIEW_FLOW_GRAPH_FEATURES + #define TBB_PREVIEW_FLOW_GRAPH_FEATURES 1 +#endif + +#include "tbb/tbb_config.h" + +#if __TBB_PREVIEW_ASYNC_MSG + +#if _MSC_VER +#pragma warning (disable: 4503) // Suppress "decorated name length exceeded, name was truncated" warning +#endif + +#include "tbb/flow_graph.h" +#include "tbb/tbb_thread.h" +#include "tbb/concurrent_queue.h" + +#include "harness.h" +#include "harness_graph.h" +#include "harness_barrier.h" + +#include // std::ostringstream +#include // std::is_base_of + +static const int USE_N = 1000; +static const int ACTIVITY_PAUSE_MS_NODE1 = 0;//500; +static const int ACTIVITY_PAUSE_MS_NODE2 = 0;//100; + +#define _TRACE_(msg) { \ + if (Verbose) { \ + std::ostringstream os; \ + os << "[TID=" << tbb::this_tbb_thread::get_id() << "] " << msg; \ + REMARK("%s\n", os.str().c_str()); \ + } \ +} + +class UserAsyncActivity // Singleton +{ +public: + static UserAsyncActivity* create(const tbb::flow::async_msg& msg, int timeoutMS) { + ASSERT(s_Activity == NULL, "created twice"); + _TRACE_( "Create UserAsyncActivity" ); + s_Activity = new UserAsyncActivity(msg, timeoutMS); + _TRACE_( "CREATED! UserAsyncActivity" ); + return s_Activity; + } + + static void destroy() { + _TRACE_( "Start UserAsyncActivity::destroy()" ); + ASSERT(s_Activity != NULL, "destroyed twice"); + s_Activity->myThread.join(); + delete s_Activity; + s_Activity = NULL; + _TRACE_( "End UserAsyncActivity::destroy()" ); + } + + static int s_Result; + +private: + static void threadFunc(UserAsyncActivity* activity) { + _TRACE_( "UserAsyncActivity::threadFunc" ); + + Harness::Sleep(activity->myTimeoutMS); + + const int result = static_cast(reinterpret_cast(activity)) & 0xFF; // just different random results + s_Result = result; + + _TRACE_( "UserAsyncActivity::threadFunc - returned result " << result ); + + activity->returnActivityResults(result); + } + + UserAsyncActivity(const tbb::flow::async_msg& msg, int timeoutMS) : myMsg(msg), myTimeoutMS(timeoutMS) + , myThread(threadFunc, this) + { + // Start local thread here... + _TRACE_( "Started AsyncActivity" ); + } + + // Will be called from working thread + void returnActivityResults(int result) { + myMsg.set(result); + } + +private: // DATA + tbb::flow::async_msg myMsg; + int myTimeoutMS; + tbb::tbb_thread myThread; + + static UserAsyncActivity* s_Activity; +}; + +UserAsyncActivity* UserAsyncActivity::s_Activity = NULL; +int UserAsyncActivity::s_Result = -1; + +class UserAsyncMsg1 : public tbb::flow::async_msg +{ +public: + typedef tbb::flow::async_msg base; + + UserAsyncMsg1() : base() {} + UserAsyncMsg1(int value) : base(value) {} + UserAsyncMsg1(const UserAsyncMsg1& msg) : base(msg) {} +}; + +struct F2_body : tbb::internal::no_assign +{ + static int s_FinalResult; + + int& myI; + bool myAlive; + + F2_body(int& i) : myI(i), myAlive(true) {} + + F2_body(const F2_body& b) : myI(b.myI), myAlive(true) {} + + ~F2_body() { + myAlive = false; + _TRACE_( "~F2_body" ); + } + + void operator () (int result) { + __TBB_ASSERT(myAlive, "dead node"); + + // Handle async activity result here + s_FinalResult = result; + _TRACE_( "F2: Got async_msg result = " << result ); + } +}; + +// static +int F2_body::s_FinalResult = -2; + +static bool testSimplestCase() { + bool bOk = true; + _TRACE_( "--- SAMPLE 1 (simple case 3-in-1: F1(A) ---> F2(T)) " ); + + for (int i = 0; i <= 2; ++i) { + _TRACE_( "CASE " << i + 1 << ": data is " << (i > 0 ? "NOT " : "") << "ready in storage" << (i > 1 ? " NO WAITING in graph" : "") ); + _TRACE_( "MAIN THREAD" ); + + { + tbb::flow::graph g; + tbb::flow::function_node< tbb::flow::continue_msg, UserAsyncMsg1 > f1( g, tbb::flow::unlimited, + [&]( tbb::flow::continue_msg ) -> UserAsyncMsg1 { + _TRACE_( "F1: Created async_msg" ); + + UserAsyncMsg1 a; + UserAsyncActivity::create(a, (i == 0 ? 0 : 1)*ACTIVITY_PAUSE_MS_NODE1); + + Harness::Sleep(ACTIVITY_PAUSE_MS_NODE2); // let activity to finish + return a; + } + ); + + + tbb::flow::function_node< int > f2( g, tbb::flow::unlimited, + F2_body(i) + ); + + make_edge(f1, f2); + f1.try_put( tbb::flow::continue_msg() ); + g.wait_for_all(); + UserAsyncActivity::destroy(); + _TRACE_( "Done UserAsyncActivity::destroy" ); + g.wait_for_all(); + _TRACE_( "Done g.wait_for_all()" ); + } + + _TRACE_( "--- THE END --- " ); + + if (F2_body::s_FinalResult >= 0 && UserAsyncActivity::s_Result == F2_body::s_FinalResult) { + _TRACE_( "CASE " << i + 1 << ": " << "PASSED" ); + } + else { + _TRACE_( "CASE " << i + 1 << ": " << "FAILED! " << UserAsyncActivity::s_Result << " != " << F2_body::s_FinalResult ); + bOk = false; + ASSERT(0, "testSimplestCase failed"); + } + } + + return bOk; +} + +// ======================================================== + +class UserAsyncActivityChaining; + +class UserAsyncMsg : public tbb::flow::async_msg +{ +public: + typedef tbb::flow::async_msg base; + + UserAsyncMsg() : base() {} + UserAsyncMsg(int value) : base(value) {} + UserAsyncMsg(const UserAsyncMsg& msg) : base(msg) {} + + // Notify AsyncActivity that it must return result because async calculation chain is over + void finalize() const __TBB_override; +}; + +class UserAsyncActivityChaining // Singleton: task queue in worker thread +{ +public: + static UserAsyncActivityChaining* instance() { + if (s_Activity == NULL) { + s_Activity = new UserAsyncActivityChaining(); + } + + return s_Activity; + } + + static void destroy() { + ASSERT(s_Activity != NULL, "destroyed twice"); + s_Activity->myThread.join(); + delete s_Activity; + s_Activity = NULL; + } + + static void finish(const UserAsyncMsg& msg) { + ASSERT(UserAsyncActivityChaining::s_Activity != NULL, "activity must be alive"); + UserAsyncActivityChaining::s_Activity->finishTaskQueue(msg); + } + + void addWork(int addValue, int timeout = 0) { + myQueue.push( MyTask(addValue, timeout) ); + } + + void finishTaskQueue(const UserAsyncMsg& msg) { + myMsg = msg; + myQueue.push( MyTask(0, 0, true) ); + } + + static int s_Result; + +private: + struct MyTask + { + MyTask(int addValue = 0, int timeout = 0, bool finishFlag = false) + : myAddValue(addValue), myTimeout(timeout), myFinishFlag(finishFlag) {} + + int myAddValue; + int myTimeout; + bool myFinishFlag; + }; + + static void threadFunc(UserAsyncActivityChaining* activity) + { + _TRACE_( "UserAsyncActivityChaining::threadFunc" ); + + for (;;) + { + // Process task queue + MyTask work; + activity->myQueue.pop(work); // Waits until it can succeed + + _TRACE_( "UserAsyncActivityChaining::threadFunc - work: add " + << work.myAddValue << " (timeout = " << work.myTimeout << ")" << (work.myFinishFlag ? " FINAL" : "") ); + + // 'finish flag' task is not real task, just end of queue flag + Harness::Sleep(work.myTimeout); + + if (work.myFinishFlag) { + break; + } + + activity->myQueueSum += work.myAddValue; + } + + s_Result = activity->myQueueSum; + _TRACE_( "UserAsyncActivityChaining::threadFunc - returned result " << activity->myQueueSum ); + + // Get result back to Flow Graph + activity->myMsg.set(activity->myQueueSum); + } + + UserAsyncActivityChaining() + : myQueueSum(0) + , myThread(threadFunc, this) + { + // Start local thread here... + _TRACE_( "Started AsyncActivityChaining" ); + } + +private: // DATA + tbb::concurrent_bounded_queue myQueue; + int myQueueSum; + UserAsyncMsg myMsg; + + tbb::tbb_thread myThread; + + static UserAsyncActivityChaining* s_Activity; +}; + +// static +UserAsyncActivityChaining* UserAsyncActivityChaining::s_Activity = NULL; +// static +int UserAsyncActivityChaining::s_Result = -4; + +// override +void UserAsyncMsg::finalize() const { + _TRACE_( "UserAsyncMsg::finalize()" ); + UserAsyncActivityChaining::finish(*this); +} + +struct F3_body : tbb::internal::no_assign +{ + static int s_FinalResult; + + int& myI; + bool myAlive; + + F3_body(int& _i) : myI(_i), myAlive(true) {} + + F3_body(const F3_body& b) : myI(b.myI), myAlive(true) {} + + ~F3_body() { + myAlive = false; + _TRACE_( "~F3_body" ); + } + + void operator () (int result) { + __TBB_ASSERT(myAlive, "dead node"); + // Handle async activity result here + s_FinalResult = result; + _TRACE_( "F3: Got async_msg result = " << result ); + } +}; + +// static +int F3_body::s_FinalResult = -8; + +static bool testChaining() { + bool bOk = true; + _TRACE_( "--- SAMPLE 2 (case with chaining: F1(A) ---> F2(A) ---> F3(T)) " ); + + for (int i = 0; i <= 2; ++i) { + _TRACE_( "CASE " << i + 1 << ": data is " << (i > 0 ? "NOT " : "") << "ready in storage" << (i > 1 ? " NO WAITING in graph" : "") ); + _TRACE_( "MAIN THREAD" ); + + tbb::flow::graph g; + tbb::flow::function_node< tbb::flow::continue_msg, UserAsyncMsg > f1( g, tbb::flow::unlimited, + [&]( tbb::flow::continue_msg ) -> UserAsyncMsg { + _TRACE_( "F1: Created UserAsyncMsg" ); + + UserAsyncMsg a; + UserAsyncActivityChaining::instance()->addWork(11, (i == 0 ? 0 : 1)*ACTIVITY_PAUSE_MS_NODE1); + + return a; + } + ); + + tbb::flow::function_node< UserAsyncMsg, UserAsyncMsg > f2( g, tbb::flow::unlimited, + [&]( UserAsyncMsg a) -> UserAsyncMsg { + _TRACE_( "F2: resend UserAsyncMsg" ); + + UserAsyncActivityChaining::instance()->addWork(22, (i == 0 ? 0 : 1)*ACTIVITY_PAUSE_MS_NODE1); + + Harness::Sleep(ACTIVITY_PAUSE_MS_NODE2); // let activity to finish + return a; + } + ); + + tbb::flow::function_node< int > f3( g, tbb::flow::unlimited, + F3_body(i) + ); + + make_edge(f1, f2); + make_edge(f2, f3); + f1.try_put( tbb::flow::continue_msg() ); + g.wait_for_all(); + + UserAsyncActivityChaining::destroy(); + _TRACE_( "Done UserAsyncActivityChaining::destroy" ); + g.wait_for_all(); + _TRACE_( "Done g.wait_for_all()" ); + + _TRACE_( "--- THE END ---" ); + + if (F3_body::s_FinalResult >= 0 && UserAsyncActivityChaining::s_Result == F3_body::s_FinalResult) { + _TRACE_( "CASE " << i + 1 << ": " << "PASSED" ); + } + else { + _TRACE_( "CASE " << i + 1 << ": " << "FAILED! " << UserAsyncActivityChaining::s_Result << " != " << F3_body::s_FinalResult ); + bOk = false; + ASSERT(0, "testChaining failed"); + } + } + + return bOk; +} + +// ======================================================== +namespace testFunctionsAvailabilityNS { + +using namespace tbb::flow; +using tbb::flow::interface10::internal::untyped_sender; +using tbb::flow::interface10::internal::untyped_receiver; + +using tbb::internal::is_same_type; +using tbb::internal::strip; +using tbb::flow::interface10::internal::wrap_tuple_elements; +using tbb::flow::interface10::internal::async_helpers; + +class A {}; // Any type (usually called 'T') +struct ImpossibleType {}; + +template +struct UserAsync_T : public async_msg { + UserAsync_T() {} + UserAsync_T(const T& t) : async_msg(t) {} +}; + +typedef UserAsync_T UserAsync_int; +typedef UserAsync_T UserAsync_float; +typedef UserAsync_T UserAsync_A; + +typedef tuple< UserAsync_A, UserAsync_float, UserAsync_int, async_msg, async_msg, async_msg, A, float, int > TypeTuple; + +static int g_CheckerCounter = 0; + +template +struct CheckerTryPut { + static ImpossibleType check( ... ); + + template + static auto check( C* p, U* q ) -> decltype(p->try_put(*q)); + + static const bool value = !is_same_type(0), 0)), ImpossibleType>::value; +}; + +template +struct CheckerMakeEdge { + static ImpossibleType checkMake( ... ); + static ImpossibleType checkRemove( ... ); + + template + static auto checkMake( N1* n1, N2* n2 ) -> decltype(tbb::flow::make_edge(*n1, *n2)); + + template + static auto checkRemove( N1* n1, N2* n2 ) -> decltype(tbb::flow::remove_edge(*n1, *n2)); + + static const bool valueMake = !is_same_type(0), static_cast(0))), ImpossibleType>::value; + static const bool valueRemove = !is_same_type(0), static_cast(0))), ImpossibleType>::value; + + __TBB_STATIC_ASSERT( valueMake == valueRemove, "make_edge() availability is NOT equal to remove_edge() availability" ); + + static const bool value = valueMake; +}; + +template +struct TypeChecker { + TypeChecker() { + ++g_CheckerCounter; + + REMARK("%d: %s -> %s: %s %s \n", g_CheckerCounter, typeid(T1).name(), typeid(T2).name(), + (bAllowed ? "YES" : "no"), (bConvertible ? " (Convertible)" : "")); + } + +// +// Check connection: function_node <-> function_node +// R E C E I V I N G T Y P E +// S 'bAllowed' | int | float | A | async_msg | async_msg | async_msg | UserAsync | UserAsync | UserAsync | +// E value | | | | | | | _int | _float | _A | +// N ------------------------------------------------------------------------------------------------------------- +// D int | Y | | | Y | | | Y | | | +// I float | | Y | | | Y | | | Y | | +// N A | | | Y | | | Y | | | Y | +// G async_msg | Y | | | Y | | | | | | +// async_msg | | Y | | | Y | | | | | +// T async_msg | | | Y | | | Y | | | | +// Y UserAsync_int | Y | | | | | | Y | | | +// P UserAsync_float | | Y | | | | | | Y | | +// E UserAsync_A | | | Y | | | | | | Y | +// + // Test make_edge() & remove_edge() availability + static const bool bAllowed = is_same_type::value + || is_same_type::filtered_type, T2>::value + || is_same_type::filtered_type>::value; + + static const bool bConvertible = bAllowed + || std::is_base_of::value + || (is_same_type::filtered_type, int>::value && is_same_type::value) + || (is_same_type::filtered_type, float>::value && is_same_type::value); + + __TBB_STATIC_ASSERT( (bAllowed == CheckerMakeEdge, function_node >::value), "invalid connection Fn -> Fn" ); + __TBB_STATIC_ASSERT( (bAllowed == CheckerMakeEdge, function_node >::value), "invalid connection Queue -> Fn" ); + + // Test make_edge() & remove_edge() availability with output_port(node&) + __TBB_STATIC_ASSERT( (bAllowed == CheckerMakeEdge( *static_cast >*>(0) ) ) >::type, + function_node >::value), "invalid connection MultuFn<0> -> Fn" ); + + __TBB_STATIC_ASSERT( (bAllowed == CheckerMakeEdge( *static_cast >*>(0) ) ) >::type, + function_node >::value), "invalid connection MultuFn<1> -> Fn" ); + + // Test untyped_sender connections + __TBB_STATIC_ASSERT( (true == CheckerMakeEdge< untyped_sender, function_node >::value), "cannot connect UntypedSender -> Fn" ); + // Test untyped_receiver connections + __TBB_STATIC_ASSERT( (true == CheckerMakeEdge< function_node, untyped_receiver >::value), "cannot connect F<.., T1> -> UntypedReceiver" ); + + // Test untyped_receiver->try_put(T2) availability + __TBB_STATIC_ASSERT( (true == CheckerTryPut::value), "untyped_receiver cannot try_put(T2)" ); + // Test receiver->try_put(T2) availability + __TBB_STATIC_ASSERT( (bConvertible == CheckerTryPut, T2>::value), "invalid availability of receiver->try_put(T2)" ); +}; + +template +struct WrappedChecker { + WrappedChecker() {} // Workaround for compilation error + + template + struct T1T2Checker : TypeChecker {}; + + typename wrap_tuple_elements< tuple_size::value, T1T2Checker, TypeTuple >::type a; +}; + +typedef wrap_tuple_elements< tuple_size::value, WrappedChecker, TypeTuple >::type Checker; + +} // namespace testFunctionsAvailabilityNS + +static void testTryPut() { + { + tbb::flow::graph g; + tbb::flow::function_node< int > f(g, tbb::flow::unlimited, [&](int) {}); + + ASSERT(f.try_put(5), "try_put(int) must return true"); + ASSERT(f.try_put(7), "try_put(int) must return true"); + + tbb::flow::async_msg a1, a2; + a1.set(5); + ASSERT(f.try_put(a1), "try_put(async_msg) must return true"); + ASSERT(f.try_put(a2), "try_put(async_msg) must return true"); + a2.set(7); + g.wait_for_all(); + } + { + tbb::flow::graph g; + typedef tbb::flow::indexer_node< int >::output_type output_type; + tbb::flow::indexer_node< int > i(g); + tbb::flow::function_node< output_type > f(g, tbb::flow::unlimited, [&](output_type) {}); + make_edge(i, f); + + ASSERT(tbb::flow::input_port<0>(i).try_put(5), "try_put(int) must return true"); + ASSERT(tbb::flow::input_port<0>(i).try_put(7), "try_put(int) must return true"); + + tbb::flow::async_msg a1(5), a2(7); + ASSERT(tbb::flow::input_port<0>(i).try_put(a1), "try_put(async_msg) must return true"); + ASSERT(tbb::flow::input_port<0>(i).try_put(a2), "try_put(async_msg) must return true"); + g.wait_for_all(); + } +} + +int TestMain() { + REMARK(" *** CHECKING FUNCTIONS: make_edge/remove_edge(node<.., T1>, node) & node->try_put(T2) ***\n"); + testFunctionsAvailabilityNS::Checker a; + const int typeTupleSize = tbb::flow::tuple_size::value; + ASSERT(testFunctionsAvailabilityNS::g_CheckerCounter == typeTupleSize*typeTupleSize, "Type checker counter value is incorrect"); + + testTryPut(); + + // NOTE: Use '-v' command line argument to get traces & remarks + tbb::task_scheduler_init init(4); + bool bOk = true; + + for (int i = 0; i < USE_N; ++i) { + if (i > 0 && i%1000 == 0) { + REPORT(" *** Starting TEST %d... ***\n", i); + } + + REMARK(" *** TEST %d ***\n", i); + bOk = bOk && testSimplestCase(); + bOk = bOk && testChaining(); + } + + _TRACE_( " *** " << USE_N << " tests: " << (bOk ? "all tests passed" : "TESTS FAILED !!!") << " ***" ); + return (bOk ? Harness::Done : Harness::Unknown); +} + +#else // __TBB_PREVIEW_ASYNC_MSG + +#include "harness.h" + +int TestMain() { + return Harness::Skipped; +} + +#endif // __TBB_PREVIEW_ASYNC_MSG diff --git a/src/tbb/src/test/test_async_node.cpp b/src/tbb/src/test/test_async_node.cpp new file mode 100644 index 00000000..e6d33a95 --- /dev/null +++ b/src/tbb/src/test/test_async_node.cpp @@ -0,0 +1,705 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "harness.h" +#include "harness_graph.h" +#include "harness_barrier.h" +#include "tbb/concurrent_queue.h" +#include "tbb/flow_graph.h" +#include "tbb/task.h" +#include "tbb/tbb_thread.h" +#include "tbb/mutex.h" +#include "tbb/compat/condition_variable" + +#include + +class minimal_type { + template + friend struct place_wrapper; + + int value; + +public: + minimal_type() : value(-1) {} + minimal_type(int v) : value(v) {} + minimal_type(const minimal_type &m) : value(m.value) { } + minimal_type &operator=(const minimal_type &m) { value = m.value; return *this; } +}; + +template +struct place_wrapper { + typedef T wrapped_type; + T value; + tbb::tbb_thread::id thread_id; + tbb::task* task_ptr; + + place_wrapper( ) : value(0) { + thread_id = tbb::this_tbb_thread::get_id(); + task_ptr = &tbb::task::self(); + } + place_wrapper( int v ) : value(v) { + thread_id = tbb::this_tbb_thread::get_id(); + task_ptr = &tbb::task::self(); + } + + place_wrapper( const place_wrapper &v ) : value(v.value), thread_id(v.thread_id), task_ptr(v.task_ptr) { } + + place_wrapper( const place_wrapper &v ) : value(v.value), thread_id(v.thread_id), task_ptr(v.task_ptr) { } +}; + +template +struct wrapper_helper { + static void check(const T1 &, const T2 &) { } + + static void copy_value(const T1 &in, T2 &out) { + out = in; + } +}; + +template +struct wrapper_helper< place_wrapper, place_wrapper > { + static void check(const place_wrapper &a, const place_wrapper &b) { + REMARK("a.task_ptr == %p != b.task_ptr == %p\n", a.task_ptr, b.task_ptr); + ASSERT( (a.thread_id != b.thread_id), "same thread used to execute adjacent nodes"); + ASSERT( (a.task_ptr != b.task_ptr), "same task used to execute adjacent nodes"); + return; + } + static void copy_value(const place_wrapper &in, place_wrapper &out) { + out.value = in.value; + } +}; + +const int NUMBER_OF_MSGS = 10; +const int UNKNOWN_NUMBER_OF_ITEMS = -1; +tbb::atomic async_body_exec_count; +tbb::atomic async_activity_processed_msg_count; +tbb::atomic end_body_exec_count; + +// queueing required in test_reset for testing of cancellation +typedef tbb::flow::async_node< int, int, tbb::flow::queueing > counting_async_node_type; +typedef counting_async_node_type::gateway_type counting_gateway_type; + +struct counting_async_body { + tbb::atomic my_async_body_exec_count; + + counting_async_body() { + my_async_body_exec_count = 0; + } + + void operator()( const int &input, counting_gateway_type& gateway) { + REMARK( "Body execution with input == %d\n", input); + ++my_async_body_exec_count; + ++async_body_exec_count; + if ( input == -1 ) { + bool result = tbb::task::self().group()->cancel_group_execution(); + REMARK( "Canceling graph execution\n" ); + ASSERT( result == true, "attempted to cancel graph twice" ); + Harness::Sleep(50); + } + gateway.try_put(input); + } +}; + +void test_reset() { + const int N = NUMBER_OF_MSGS; + async_body_exec_count = 0; + + tbb::flow::graph g; + counting_async_node_type a(g, tbb::flow::serial, counting_async_body() ); + + const int R = 3; + std::vector< harness_counting_receiver > r(R, harness_counting_receiver(g)); + + for (int i = 0; i < R; ++i) { +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + tbb::flow::make_edge(a, r[i]); +#else + tbb::flow::make_edge( tbb::flow::output_port<0>(a), r[i] ); +#endif + } + + REMARK( "One body execution\n" ); + a.try_put(-1); + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + // should be canceled with only 1 item reaching the async_body and the counting receivers + // and N items left in the node's queue + ASSERT( g.is_cancelled() == true, "task group not canceled" ); + + counting_async_body b1 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == int(b1.my_async_body_exec_count), "body and global body counts are different" ); + ASSERT( int(async_body_exec_count) == 1, "global body execution count not 1" ); + for (int i = 0; i < R; ++i) { + ASSERT( int(r[i].my_count) == 1, "counting receiver count not 1" ); + } + + // should clear the async_node queue, but retain its local count at 1 and keep all edges + g.reset(tbb::flow::rf_reset_protocol); + + REMARK( "N body executions\n" ); + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + ASSERT( g.is_cancelled() == false, "task group not canceled" ); + + // a total of N+1 items should have passed through the node body + // the local body count should also be N+1 + // and the counting receivers should all have a count of N+1 + counting_async_body b2 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == int(b2.my_async_body_exec_count), "local and global body execution counts are different" ); + REMARK( "async_body_exec_count==%d\n", int(async_body_exec_count) ); + ASSERT( int(async_body_exec_count) == N+1, "globcal body execution count not N+1" ); + for (int i = 0; i < R; ++i) { + ASSERT( int(r[i].my_count) == N+1, "counting receiver has not received N+1 items" ); + } + + REMARK( "N body executions with new bodies\n" ); + // should clear the async_node queue and reset its local count to 0, but keep all edges + g.reset(tbb::flow::rf_reset_bodies); + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + ASSERT( g.is_cancelled() == false, "task group not canceled" ); + + // a total of 2N+1 items should have passed through the node body + // the local body count should be N + // and the counting receivers should all have a count of 2N+1 + counting_async_body b3 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == 2*N+1, "global body execution count not 2N+1" ); + ASSERT( int(b3.my_async_body_exec_count) == N, "local body execution count not N" ); + for (int i = 0; i < R; ++i) { + ASSERT( int(r[i].my_count) == 2*N+1, "counting receiver has not received 2N+1 items" ); + } + + // should clear the async_node queue and keep its local count at N and remove all edges + REMARK( "N body executions with no edges\n" ); + g.reset(tbb::flow::rf_clear_edges); + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + ASSERT( g.is_cancelled() == false, "task group not canceled" ); + + // a total of 3N+1 items should have passed through the node body + // the local body count should now be 2*N + // and the counting receivers should remain at a count of 2N+1 + counting_async_body b4 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == 3*N+1, "global body execution count not 3N+1" ); + ASSERT( int(b4.my_async_body_exec_count) == 2*N, "local body execution count not 2N" ); + for (int i = 0; i < R; ++i) { + ASSERT( int(r[i].my_count) == 2*N+1, "counting receiver has not received 2N+1 items" ); + } + + // put back 1 edge to receiver 0 + REMARK( "N body executions with 1 edge\n" ); +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + tbb::flow::make_edge(a, r[0]); +#else + tbb::flow::make_edge( tbb::flow::output_port<0>(a), r[0] ); +#endif + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + ASSERT( g.is_cancelled() == false, "task group not canceled" ); + + // a total of 4N+1 items should have passed through the node body + // the local body count should now be 3*N + // and all of the counting receivers should remain at a count of 2N+1, except r[0] which should be 3N+1 + counting_async_body b5 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == 4*N+1, "global body execution count not 4N+1" ); + ASSERT( int(b5.my_async_body_exec_count) == 3*N, "local body execution count not 3N" ); + ASSERT( int(r[0].my_count) == 3*N+1, "counting receiver has not received 3N+1 items" ); + for (int i = 1; i < R; ++i) { + ASSERT( int(r[i].my_count) == 2*N+1, "counting receiver has not received 2N+1 items" ); + } + + // should clear the async_node queue and keep its local count at N and remove all edges + REMARK( "N body executions with no edges and new body\n" ); + g.reset(static_cast(tbb::flow::rf_reset_bodies|tbb::flow::rf_clear_edges)); + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + ASSERT( g.is_cancelled() == false, "task group not canceled" ); + + // a total of 4N+1 items should have passed through the node body + // the local body count should now be 3*N + // and all of the counting receivers should remain at a count of 2N+1, except r[0] which should be 3N+1 + counting_async_body b6 = tbb::flow::copy_body(a); + ASSERT( int(async_body_exec_count) == 5*N+1, "global body execution count not 5N+1" ); + ASSERT( int(b6.my_async_body_exec_count) == N, "local body execution count not N" ); + ASSERT( int(r[0].my_count) == 3*N+1, "counting receiver has not received 3N+1 items" ); + for (int i = 1; i < R; ++i) { + ASSERT( int(r[i].my_count) == 2*N+1, "counting receiver has not received 2N+1 items" ); + } +} + +template< typename Input, typename Output > +class async_activity : NoAssign { +public: + typedef Input input_type; + typedef Output output_type; + typedef tbb::flow::async_node< input_type, output_type > async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + + struct work_type { + input_type input; + gateway_type* gateway; + }; + + class ServiceThreadBody { + public: + ServiceThreadBody( async_activity* activity ) : my_activity( activity ) {} + + void operator()() { + my_activity->process(); + } + private: + async_activity* my_activity; + }; + + async_activity(int expected_items, bool deferred = false, int sleep_time = 50) + : my_expected_items(expected_items), my_sleep_time(sleep_time) { + is_active = !deferred; + my_quit = false; + tbb::tbb_thread( ServiceThreadBody( this ) ).swap( my_service_thread ); + } + +private: + + async_activity( const async_activity& ) + : my_expected_items(UNKNOWN_NUMBER_OF_ITEMS), my_sleep_time(0) { + is_active = true; + } + +public: + ~async_activity() { + stop(); + my_service_thread.join(); + } + + void submit( const input_type &input, gateway_type& gateway ) { + work_type work = { input, &gateway}; + my_work_queue.push( work ); + } + + void process() { + do { + work_type work; + if( is_active && my_work_queue.try_pop( work ) ) { + Harness::Sleep(my_sleep_time); + ++async_activity_processed_msg_count; + output_type output; + wrapper_helper::copy_value(work.input, output); + wrapper_helper::check(work.input, output); + work.gateway->try_put(output); + if ( my_expected_items == UNKNOWN_NUMBER_OF_ITEMS || + int(async_activity_processed_msg_count) == my_expected_items ) { + work.gateway->release_wait(); + } + } + } while( my_quit == false || !my_work_queue.empty()); + } + + void stop() { + my_quit = true; + } + + void activate() { + is_active = true; + } + + bool should_reserve_each_time() { + if ( my_expected_items == UNKNOWN_NUMBER_OF_ITEMS ) + return true; + else + return false; + } + +private: + + const int my_expected_items; + const int my_sleep_time; + tbb::atomic< bool > is_active; + + tbb::concurrent_queue< work_type > my_work_queue; + + tbb::atomic< bool > my_quit; + + tbb::tbb_thread my_service_thread; +}; + +template +struct basic_test { + typedef Input input_type; + typedef Output output_type; + typedef tbb::flow::async_node< input_type, output_type > async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + + class start_body_type { + typedef Input input_type; + public: + input_type operator()( int input ) { + return input_type(input); + } + }; + +#if !__TBB_CPP11_LAMBDAS_PRESENT + class async_body_type { + typedef Input input_type; + typedef Output output_type; + typedef tbb::flow::async_node< input_type, output_type > async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + public: + typedef async_activity async_activity_type; + + async_body_type( async_activity_type* aa ) : my_async_activity( aa ) { } + + async_body_type( const async_body_type& other ) : my_async_activity( other.my_async_activity ) { } + + void operator()( const input_type &input, gateway_type& gateway ) { + ++async_body_exec_count; + my_async_activity->submit( input, gateway); + if ( my_async_activity->should_reserve_each_time() ) + gateway.reserve_wait(); + } + + private: + async_activity_type* my_async_activity; + }; +#endif + + class end_body_type { + typedef Output output_type; + public: + void operator()( const output_type &input ) { + ++end_body_exec_count; + output_type output; + wrapper_helper::check(input, output); + } + }; + + basic_test() {} + +public: + + static int run(int async_expected_items = UNKNOWN_NUMBER_OF_ITEMS) { + async_activity my_async_activity(async_expected_items); + tbb::flow::graph g; + tbb::flow::function_node< int, input_type > start_node( g, tbb::flow::unlimited, start_body_type() ); +#if __TBB_CPP11_LAMBDAS_PRESENT + async_node_type offload_node(g, tbb::flow::unlimited, [&] (const input_type &input, gateway_type& gateway) { + ++async_body_exec_count; + my_async_activity.submit(input, gateway); + if(my_async_activity.should_reserve_each_time()) + gateway.reserve_wait(); + } ); +#else + async_node_type offload_node( g, tbb::flow::unlimited, async_body_type( &my_async_activity ) ); +#endif + + tbb::flow::function_node< output_type > end_node( g, tbb::flow::unlimited, end_body_type() ); + + tbb::flow::make_edge( start_node, offload_node ); +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + tbb::flow::make_edge( offload_node, end_node ); +#else + tbb::flow::make_edge( tbb::flow::output_port<0>(offload_node), end_node ); +#endif + async_body_exec_count = 0; + async_activity_processed_msg_count = 0; + end_body_exec_count = 0; + + if (async_expected_items != UNKNOWN_NUMBER_OF_ITEMS ) { + offload_node.gateway().reserve_wait(); + } + for (int i = 0; i < NUMBER_OF_MSGS; ++i) { + start_node.try_put(i); + } + g.wait_for_all(); + ASSERT( async_body_exec_count == NUMBER_OF_MSGS, "AsyncBody processed wrong number of signals" ); + ASSERT( async_activity_processed_msg_count == NUMBER_OF_MSGS, "AsyncActivity processed wrong number of signals" ); + ASSERT( end_body_exec_count == NUMBER_OF_MSGS, "EndBody processed wrong number of signals"); + REMARK("async_body_exec_count == %d == async_activity_processed_msg_count == %d == end_body_exec_count == %d\n", + int(async_body_exec_count), int(async_activity_processed_msg_count), int(end_body_exec_count)); + return Harness::Done; + } + +}; + +int test_copy_ctor() { + const int N = NUMBER_OF_MSGS; + async_body_exec_count = 0; + + tbb::flow::graph g; + + harness_counting_receiver r1(g); + harness_counting_receiver r2(g); + + counting_async_node_type a(g, tbb::flow::unlimited, counting_async_body() ); + counting_async_node_type b(a); +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + tbb::flow::make_edge(a, r1); + tbb::flow::make_edge(b, r2); +#else + tbb::flow::make_edge(tbb::flow::output_port<0>(a), r1); + tbb::flow::make_edge(tbb::flow::output_port<0>(b), r2); +#endif + + for (int i = 0; i < N; ++i) { + a.try_put(i); + } + g.wait_for_all(); + + REMARK("async_body_exec_count = %d\n", int(async_body_exec_count)); + REMARK("r1.my_count == %d and r2.my_count = %d\n", int(r1.my_count), int(r2.my_count)); + ASSERT( int(async_body_exec_count) == NUMBER_OF_MSGS, "AsyncBody processed wrong number of signals" ); + ASSERT( int(r1.my_count) == N, "counting receiver r1 has not received N items" ); + ASSERT( int(r2.my_count) == 0, "counting receiver r2 has not received 0 items" ); + + for (int i = 0; i < N; ++i) { + b.try_put(i); + } + g.wait_for_all(); + + REMARK("async_body_exec_count = %d\n", int(async_body_exec_count)); + REMARK("r1.my_count == %d and r2.my_count = %d\n", int(r1.my_count), int(r2.my_count)); + ASSERT( int(async_body_exec_count) == 2*NUMBER_OF_MSGS, "AsyncBody processed wrong number of signals" ); + ASSERT( int(r1.my_count) == N, "counting receiver r1 has not received N items" ); + ASSERT( int(r2.my_count) == N, "counting receiver r2 has not received N items" ); + return Harness::Done; +} + +tbb::atomic main_tid_count; + +template +struct spin_test { + typedef Input input_type; + typedef Output output_type; + typedef tbb::flow::async_node< input_type, output_type > async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + + class start_body_type { + typedef Input input_type; + public: + input_type operator()( int input ) { + return input_type(input); + } + }; + +#if !__TBB_CPP11_LAMBDAS_PRESENT + class async_body_type { + typedef Input input_type; + typedef Output output_type; + typedef tbb::flow::async_node< input_type, output_type > async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + public: + typedef async_activity async_activity_type; + + async_body_type( async_activity_type* aa ) : my_async_activity( aa ) { } + + async_body_type( const async_body_type& other ) : my_async_activity( other.my_async_activity ) { } + + void operator()(const input_type &input, gateway_type& gateway) { + ++async_body_exec_count; + my_async_activity->submit(input, gateway); + if(my_async_activity->should_reserve_each_time()) + gateway.reserve_wait(); + } + + private: + async_activity_type* my_async_activity; + }; +#endif + + class end_body_type { + typedef Output output_type; + tbb::tbb_thread::id my_main_tid; + Harness::SpinBarrier *my_barrier; + public: + end_body_type(tbb::tbb_thread::id t, Harness::SpinBarrier &b) : my_main_tid(t), my_barrier(&b) { } + + void operator()( const output_type & ) { + ++end_body_exec_count; + if (tbb::this_tbb_thread::get_id() == my_main_tid) { + ++main_tid_count; + } + my_barrier->timed_wait_noerror(10); + } + }; + + spin_test() {} + + static int run(int nthreads, int async_expected_items = UNKNOWN_NUMBER_OF_ITEMS) { + async_activity my_async_activity(async_expected_items, false, 0); + Harness::SpinBarrier spin_barrier(nthreads); + tbb::flow::graph g; + tbb::flow::function_node< int, input_type > start_node( g, tbb::flow::unlimited, start_body_type() ); +#if __TBB_CPP11_LAMBDAS_PRESENT + async_node_type offload_node(g, tbb::flow::unlimited, [&](const input_type &input, gateway_type& gateway) { + ++async_body_exec_count; + my_async_activity.submit(input, gateway); + if(my_async_activity.should_reserve_each_time()) + gateway.reserve_wait(); + }); +#else + async_node_type offload_node( g, tbb::flow::unlimited, async_body_type( &my_async_activity ) ); +#endif + tbb::flow::function_node< output_type > end_node( g, tbb::flow::unlimited, end_body_type(tbb::this_tbb_thread::get_id(), spin_barrier) ); + tbb::flow::make_edge( start_node, offload_node ); +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + tbb::flow::make_edge( offload_node, end_node ); +#else + tbb::flow::make_edge( tbb::flow::output_port<0>(offload_node), end_node ); +#endif + async_body_exec_count = 0; + async_activity_processed_msg_count = 0; + end_body_exec_count = 0; + main_tid_count = 0; + + if (async_expected_items != UNKNOWN_NUMBER_OF_ITEMS ) { + offload_node.gateway().reserve_wait(); + } + for (int i = 0; i < nthreads*NUMBER_OF_MSGS; ++i) { + start_node.try_put(i); + } + g.wait_for_all(); + ASSERT( async_body_exec_count == nthreads*NUMBER_OF_MSGS, "AsyncBody processed wrong number of signals" ); + ASSERT( async_activity_processed_msg_count == nthreads*NUMBER_OF_MSGS, "AsyncActivity processed wrong number of signals" ); + ASSERT( end_body_exec_count == nthreads*NUMBER_OF_MSGS, "EndBody processed wrong number of signals"); + ASSERT_WARNING( main_tid_count != 0, "Main thread did not participate in end_body tasks"); + REMARK("async_body_exec_count == %d == async_activity_processed_msg_count == %d == end_body_exec_count == %d\n", + int(async_body_exec_count), int(async_activity_processed_msg_count), int(end_body_exec_count)); + return Harness::Done; + } + +}; + +void test_for_spin_avoidance() { + spin_test::run(4); +} + +template< typename Input, typename Output > +int run_tests() { + basic_test::run(); + basic_test::run(NUMBER_OF_MSGS); + basic_test, place_wrapper >::run(); + basic_test, place_wrapper >::run(NUMBER_OF_MSGS); + return Harness::Done; +} + +#include "tbb/parallel_for.h" +template +class equeueing_on_inner_level { + typedef Input input_type; + typedef Output output_type; + typedef async_activity async_activity_type; + typedef tbb::flow::async_node async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + + class start_body_type { + public: + input_type operator() ( int input ) { + return input_type( input); + } + }; + + class async_body_type { + public: + async_body_type( async_activity_type& activity ) : my_async_activity(&activity) {} + + void operator() ( const input_type &input, gateway_type& gateway ) { + gateway.reserve_wait(); + my_async_activity->submit( input, gateway ); + } + private: + async_activity_type* my_async_activity; + }; + + class end_body_type { + public: + void operator()( output_type ) {} + }; + + class body_graph_with_async { + public: + body_graph_with_async( Harness::SpinBarrier& barrier, async_activity_type& activity ) + : spin_barrier(&barrier), my_async_activity(&activity) {} + + void operator()(int) const { + tbb::flow::graph g; + tbb::flow::function_node< int, input_type > start_node( g, tbb::flow::unlimited, start_body_type() ); + + async_node_type offload_node( g, tbb::flow::unlimited, async_body_type( *my_async_activity ) ); + + tbb::flow::function_node< output_type > end_node( g, tbb::flow::unlimited, end_body_type() ); + + tbb::flow::make_edge( start_node, offload_node ); + tbb::flow::make_edge( offload_node, end_node ); + + start_node.try_put(1); + + spin_barrier->wait(); + + my_async_activity->activate(); + + g.wait_for_all(); + } + + private: + Harness::SpinBarrier* spin_barrier; + async_activity_type* my_async_activity; + }; + + +public: + static int run () + { + const int nthreads = tbb::this_task_arena::max_concurrency(); + Harness::SpinBarrier spin_barrier( nthreads ); + + async_activity_type my_async_activity( UNKNOWN_NUMBER_OF_ITEMS, true ); + + tbb::parallel_for( 0, nthreads, body_graph_with_async( spin_barrier, my_async_activity ) ); + return Harness::Done; + } +}; + +int run_test_equeueing_on_inner_level() { + equeueing_on_inner_level::run(); + return Harness::Done; +} + +int TestMain() { + tbb::task_scheduler_init init(4); + run_tests(); + run_tests(); + run_tests(); + + lightweight_testing::test(NUMBER_OF_MSGS); + + test_reset(); + test_copy_ctor(); + test_for_spin_avoidance(); + run_test_equeueing_on_inner_level(); + return Harness::Done; +} + diff --git a/src/tbb/src/test/test_atomic.cpp b/src/tbb/src/test/test_atomic.cpp new file mode 100644 index 00000000..065df696 --- /dev/null +++ b/src/tbb/src/test/test_atomic.cpp @@ -0,0 +1,1601 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "harness_defs.h" + +#if __TBB_TEST_SKIP_PIC_MODE || (__TBB_TEST_SKIP_GCC_BUILTINS_MODE && __TBB_TEST_SKIP_ICC_BUILTINS_MODE) +#include "harness.h" +int TestMain() { + REPORT("Known issue: %s\n", + __TBB_TEST_SKIP_PIC_MODE? "PIC mode is not supported" : "Compiler builtins for atomic operations aren't available"); + return Harness::Skipped; +} +#else + +// Put tbb/atomic.h first, so if it is missing a prerequisite header, we find out about it. +// The tests here do *not* test for atomicity, just serial correctness. */ + +#include "tbb/atomic.h" +#include "harness_assert.h" +#include // memcmp +#include "tbb/aligned_space.h" +#include //for placement new + +using std::memcmp; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Unary minus operator applied to unsigned type, result still unsigned + // Constant conditional expression + #pragma warning( disable: 4127 4310 ) +#endif + +#if __TBB_GCC_STRICT_ALIASING_BROKEN + #pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif + +enum LoadStoreExpression { + UseOperators, + UseImplicitAcqRel, + UseExplicitFullyFenced, + UseExplicitAcqRel, + UseExplicitRelaxed, + UseGlobalHelperFullyFenced, + UseGlobalHelperAcqRel, + UseGlobalHelperRelaxed +}; + +//! Structure that holds an atomic and some guard bytes around it. +template +struct TestStruct { + typedef unsigned char byte_type; + T prefix; + tbb::atomic counter; + T suffix; + TestStruct( T i ) { + ASSERT( sizeof(*this)==3*sizeof(T), NULL ); + for (size_t j = 0; j < sizeof(T); ++j) { + reinterpret_cast(&prefix)[j] = byte_type(0x11*(j+1)); + reinterpret_cast(&suffix)[sizeof(T)-j-1] = byte_type(0x11*(j+1)); + } + if ( E == UseOperators ) + counter = i; + else if ( E == UseExplicitRelaxed ) + counter.template store(i); + else + tbb::store( counter, i ); + } + ~TestStruct() { + // Check for writes outside the counter. + for (size_t j = 0; j < sizeof(T); ++j) { + ASSERT( reinterpret_cast(&prefix)[j] == byte_type(0x11*(j+1)), NULL ); + ASSERT( reinterpret_cast(&suffix)[sizeof(T)-j-1] == byte_type(0x11*(j+1)), NULL ); + } + } + static tbb::atomic gCounter; +}; + +// A global variable of type tbb::atomic<> +template tbb::atomic TestStruct::gCounter; + +//! Test compare_and_swap template members of class atomic for memory_semantics=M +template +void TestCompareAndSwapWithExplicitOrdering( T i, T j, T k ) { + ASSERT( i!=k && i!=j, "values must be distinct" ); + // Test compare_and_swap that should fail + TestStruct x(i); + T old = x.counter.template compare_and_swap( j, k ); + ASSERT( old==i, NULL ); + ASSERT( x.counter==i, "old value not retained" ); + // Test compare and swap that should succeed + old = x.counter.template compare_and_swap( j, i ); + ASSERT( old==i, NULL ); + ASSERT( x.counter==j, "value not updated?" ); +} + +//! i, j, k must be different values +template +void TestCompareAndSwap( T i, T j, T k ) { + ASSERT( i!=k && i!=j, "values must be distinct" ); + // Test compare_and_swap that should fail + TestStruct x(i); + T old = x.counter.compare_and_swap( j, k ); + ASSERT( old==i, NULL ); + ASSERT( x.counter==i, "old value not retained" ); + // Test compare and swap that should succeed + old = x.counter.compare_and_swap( j, i ); + ASSERT( old==i, NULL ); + if( x.counter==i ) { + ASSERT( x.counter==j, "value not updated?" ); + } else { + ASSERT( x.counter==j, "value trashed" ); + } + // Check that atomic global variables work + TestStruct::gCounter = i; + old = TestStruct::gCounter.compare_and_swap( j, i ); + ASSERT( old==i, NULL ); + ASSERT( TestStruct::gCounter==j, "value not updated?" ); + TestCompareAndSwapWithExplicitOrdering(i,j,k); + TestCompareAndSwapWithExplicitOrdering(i,j,k); + TestCompareAndSwapWithExplicitOrdering(i,j,k); + TestCompareAndSwapWithExplicitOrdering(i,j,k); +} + +//! memory_semantics variation on TestFetchAndStore +template +void TestFetchAndStoreWithExplicitOrdering( T i, T j ) { + ASSERT( i!=j, "values must be distinct" ); + TestStruct x(i); + T old = x.counter.template fetch_and_store( j ); + ASSERT( old==i, NULL ); + ASSERT( x.counter==j, NULL ); +} + +//! i and j must be different values +template +void TestFetchAndStore( T i, T j ) { + ASSERT( i!=j, "values must be distinct" ); + TestStruct x(i); + T old = x.counter.fetch_and_store( j ); + ASSERT( old==i, NULL ); + ASSERT( x.counter==j, NULL ); + // Check that atomic global variables work + TestStruct::gCounter = i; + old = TestStruct::gCounter.fetch_and_store( j ); + ASSERT( old==i, NULL ); + ASSERT( TestStruct::gCounter==j, "value not updated?" ); + TestFetchAndStoreWithExplicitOrdering(i,j); + TestFetchAndStoreWithExplicitOrdering(i,j); + TestFetchAndStoreWithExplicitOrdering(i,j); + TestFetchAndStoreWithExplicitOrdering(i,j); +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // conversion from to , possible loss of data + // the warning seems a complete nonsense when issued for e.g. short+=short + #pragma warning( disable: 4244 ) +#endif + +//! Test fetch_and_add members of class atomic for memory_semantics=M +template +void TestFetchAndAddWithExplicitOrdering( T i ) { + TestStruct x(i); + T actual; + T expected = i; + + // Test fetch_and_add member template + for( int j=0; j<10; ++j ) { + actual = x.counter.fetch_and_add(j); + ASSERT( actual==expected, NULL ); + expected += j; + } + for( int j=0; j<10; ++j ) { + actual = x.counter.fetch_and_add(-j); + ASSERT( actual==expected, NULL ); + expected -= j; + } + + // Test fetch_and_increment member template + ASSERT( x.counter==i, NULL ); + actual = x.counter.template fetch_and_increment(); + ASSERT( actual==i, NULL ); + ASSERT( x.counter==T(i+1), NULL ); + + // Test fetch_and_decrement member template + actual = x.counter.template fetch_and_decrement(); + ASSERT( actual==T(i+1), NULL ); + ASSERT( x.counter==i, NULL ); +} + +//! Test fetch_and_add and related operators +template +void TestFetchAndAdd( T i ) { + TestStruct x(i); + T value; + value = ++x.counter; + ASSERT( value==T(i+1), NULL ); + value = x.counter++; + ASSERT( value==T(i+1), NULL ); + value = x.counter--; + ASSERT( value==T(i+2), NULL ); + value = --x.counter; + ASSERT( value==i, NULL ); + T actual; + T expected = i; + for( int j=-100; j<=100; ++j ) { + expected += j; + actual = x.counter += j; + ASSERT( actual==expected, NULL ); + } + for( int j=-100; j<=100; ++j ) { + expected -= j; + actual = x.counter -= j; + ASSERT( actual==expected, NULL ); + } + // Test fetch_and_increment + ASSERT( x.counter==i, NULL ); + actual = x.counter.fetch_and_increment(); + ASSERT( actual==i, NULL ); + ASSERT( x.counter==T(i+1), NULL ); + + // Test fetch_and_decrement + actual = x.counter.fetch_and_decrement(); + ASSERT( actual==T(i+1), NULL ); + ASSERT( x.counter==i, NULL ); + x.counter = i; + ASSERT( x.counter==i, NULL ); + + // Check that atomic global variables work + TestStruct::gCounter = i; + value = TestStruct::gCounter.fetch_and_add( 42 ); + expected = i+42; + ASSERT( value==i, NULL ); + ASSERT( TestStruct::gCounter==expected, "value not updated?" ); + TestFetchAndAddWithExplicitOrdering(i); + TestFetchAndAddWithExplicitOrdering(i); + TestFetchAndAddWithExplicitOrdering(i); + TestFetchAndAddWithExplicitOrdering(i); +} + +//! A type with unknown size. +class IncompleteType; + +void TestFetchAndAdd( IncompleteType* ) { + // There are no fetch-and-add operations on a IncompleteType*. +} +void TestFetchAndAdd( void* ) { + // There are no fetch-and-add operations on a void*. +} + +void TestFetchAndAdd( bool ) { + // There are no fetch-and-add operations on a bool. +} + +template +void TestConst( T i ) { + // Try const + const TestStruct x(i); + ASSERT( memcmp( &i, &x.counter, sizeof(T) )==0, "write to atomic broken?" ); + ASSERT( x.counter==i, "read of atomic broken?" ); + const TestStruct y(i); + ASSERT( memcmp( &i, &y.counter, sizeof(T) )==0, "relaxed write to atomic broken?" ); + ASSERT( tbb::load(y.counter) == i, "relaxed read of atomic broken?" ); + const TestStruct z(i); + ASSERT( memcmp( &i, &z.counter, sizeof(T) )==0, "sequentially consistent write to atomic broken?" ); + ASSERT( z.counter.template load() == i, "sequentially consistent read of atomic broken?" ); +} + +#include "harness.h" + +#include + +//TODO: consider moving it to separate file, and unify with one in examples command line interface +template +std::string to_string(const T& a){ + std::stringstream str; str < + struct test_initialization_fixture{ + typedef tbb::atomic atomic_t; + tbb::aligned_space non_zeroed_storage; + enum {fill_value = 0xFF }; + test_initialization_fixture(){ + memset(static_cast(non_zeroed_storage.begin()),fill_value, + sizeof(non_zeroed_storage)); + ASSERT( char(fill_value)==*(reinterpret_cast(non_zeroed_storage.begin())) + ,"failed to fill the storage; memset error?"); + } + //TODO: consider move it to destructor, even in a price of UB + void tear_down(){ + non_zeroed_storage.begin()->~atomic_t(); + } + }; + + template + struct TestValueInitialization : test_initialization_fixture{ + void operator()(){ + typedef typename test_initialization_fixture::atomic_t atomic_type; + //please note that explicit braces below are needed to get zero initialization. + //in C++11, 8.5 Initializers [dcl.init], see paragraphs 10,7,5 + new (this->non_zeroed_storage.begin()) atomic_type(); + //TODO: add use of KNOWN_ISSUE macro on SunCC 5.11 + #if !__SUNPRO_CC || __SUNPRO_CC > 0x5110 + //TODO: add printing of typename to the assertion + ASSERT(char(0)==*(reinterpret_cast(this->non_zeroed_storage.begin())) + ,("value initialization for tbb::atomic should do zero initialization; " + "actual value:"+to_string(this->non_zeroed_storage.begin()->load())).c_str()); + #endif + this->tear_down(); + }; + }; + + template + struct TestDefaultInitialization : test_initialization_fixture{ + void operator ()(){ + typedef typename test_initialization_fixture::atomic_t atomic_type; + new (this->non_zeroed_storage.begin()) atomic_type; + ASSERT( char(this->fill_value)==*(reinterpret_cast(this->non_zeroed_storage.begin())) + ,"default initialization for atomic should do no initialization"); + this->tear_down(); + } + }; +# if __TBB_ATOMIC_CTORS + template + struct TestDirectInitialization : test_initialization_fixture { + void operator()(T i){ + typedef typename test_initialization_fixture::atomic_t atomic_type; + new (this->non_zeroed_storage.begin()) atomic_type(i); + ASSERT(i == this->non_zeroed_storage.begin()->load() + ,("tbb::atomic initialization failed; " + "value:"+to_string(this->non_zeroed_storage.begin()->load())+ + "; expected:"+to_string(i)).c_str()); + this->tear_down(); + } + }; +# endif +} +template +void TestValueInitialization(){ + initialization_tests::TestValueInitialization()(); +} +template +void TestDefaultInitialization(){ + initialization_tests::TestDefaultInitialization()(); +} + +#if __TBB_ATOMIC_CTORS +template +void TestDirectInitialization(T i){ + initialization_tests::TestDirectInitialization()(i); +} +//TODO: it would be great to have constructor doing dynamic initialization of local atomic objects implicitly (with zero?), +// but do no dynamic initializations by default for static objects +namespace test_constexpr_initialization_helper { + struct white_box_ad_hoc_type { + int _int; + constexpr white_box_ad_hoc_type(int a =0) : _int(a) {}; + constexpr operator int() const { return _int; } + }; +} +//some white boxing +namespace tbb { namespace internal { + template<> + struct atomic_impl: atomic_impl { + atomic_impl() = default; + constexpr atomic_impl(test_constexpr_initialization_helper::white_box_ad_hoc_type value):atomic_impl(value){} + constexpr operator int() const { return this->my_storage.my_value; } + }; +}} + +//TODO: make this a parameterized macro +void TestConstExprInitializationIsTranslationTime(){ + const char* ct_init_failed_msg = "translation time init failed?"; + typedef tbb::atomic atomic_t; + constexpr atomic_t a(8); + ASSERT(a == 8,ct_init_failed_msg); + +#if !__TBB_CONSTEXPR_MEMBER_FUNCTION_BROKEN + constexpr tbb::atomic ct_atomic(10); + //for some unknown reason clang does not managed to enum syntax +#if __clang__ + constexpr int ct_atomic_value_ten = (int)ct_atomic; +#else + enum {ct_atomic_value_ten = (int)ct_atomic}; +#endif + __TBB_STATIC_ASSERT(ct_atomic_value_ten == 10, "translation time init failed?"); + ASSERT(ct_atomic_value_ten == 10,ct_init_failed_msg); + int array[ct_atomic_value_ten]; + ASSERT(Harness::array_length(array) == 10,ct_init_failed_msg); +#endif //__TBB_CONSTEXPR_MEMBER_FUNCTION_BROKEN +} + +#include +#include +namespace TestConstExprInitializationOfGlobalObjectsHelper{ + struct static_objects_dynamic_init_order_tester { + static int order_hash; + template struct nth { + nth(){ order_hash = (order_hash<<4)+N; } + }; + + static nth<2> second; + static nth<3> third; + }; + + int static_objects_dynamic_init_order_tester::order_hash=1; + static_objects_dynamic_init_order_tester::nth<2> static_objects_dynamic_init_order_tester::second; + static_objects_dynamic_init_order_tester::nth<3> static_objects_dynamic_init_order_tester::third; + + void TestStaticsDynamicInitializationOrder(){ + ASSERT(static_objects_dynamic_init_order_tester::order_hash==0x123,"Statics dynamic initialization order is broken? "); + } + + template + void TestStaticInit(); + + namespace auto_registered_tests_helper { + template + struct type_name ; + + #define REGISTER_TYPE_NAME(T) \ + namespace auto_registered_tests_helper{ \ + template<> \ + struct type_name { \ + static const char* name; \ + }; \ + const char* type_name::name = #T; \ + } \ + + typedef void (* p_test_function_type)(); + static std::vector const_expr_tests; + + template + struct registration{ + registration(){const_expr_tests.push_back(&TestStaticInit);} + }; + } + //according to ISO C++11 [basic.start.init], static data fields of class template have unordered + //initialization unless it is an explicit specialization + template + struct tester; + + #define TESTER_SPECIALIZATION(T,ct_value) \ + template<> \ + struct tester { \ + struct static_before; \ + static bool result; \ + static static_before static_before_; \ + static tbb::atomic static_atomic; \ + \ + static auto_registered_tests_helper::registration registered; \ + }; \ + bool tester::result = false; \ + \ + struct tester::static_before { \ + static_before(){ result = (static_atomic==ct_value); } \ + } ; \ + \ + tester::static_before tester::static_before_; \ + tbb::atomic tester::static_atomic(ct_value); \ + \ + auto_registered_tests_helper::registration tester::registered; \ + REGISTER_TYPE_NAME(T) \ + + template + void TestStaticInit(){ + //TODO: add printing of values to the assertion + std::string type_name = auto_registered_tests_helper::type_name::name; + ASSERT(tester::result,("Static initialization failed for atomic " + type_name).c_str()); + } + + void CallExprInitTests(){ +# if __TBB_STATIC_CONSTEXPR_INIT_BROKEN + REPORT("Known issue: Compile-time initialization fails for static tbb::atomic variables\n"); +# else + using namespace auto_registered_tests_helper; + for (size_t i =0; i as in TestMain +} + +void TestConstExprInitializationOfGlobalObjects(){ + //first assert that assumption the test based on are correct + TestConstExprInitializationOfGlobalObjectsHelper::TestStaticsDynamicInitializationOrder(); + TestConstExprInitializationOfGlobalObjectsHelper::CallExprInitTests(); +} +#endif //__TBB_ATOMIC_CTORS +template +void TestOperations( T i, T j, T k ) { + TestValueInitialization(); + TestDefaultInitialization(); +# if __TBB_ATOMIC_CTORS + TestConstExprInitializationIsTranslationTime(); + TestDirectInitialization(i); + TestDirectInitialization(j); + TestDirectInitialization(k); +# endif + TestConst(i); + TestCompareAndSwap(i,j,k); + TestFetchAndStore(i,k); // Pass i,k instead of i,j, because callee requires two distinct values. +} + +template +void TestParallel( const char* name ); + +bool ParallelError; + +template +struct AlignmentChecker { + char c; + tbb::atomic i; +}; + +//TODO: candidate for test_compiler? +template +void TestAlignment( const char* name ) { + AlignmentChecker ac; + tbb::atomic x; + x = T(0); + bool is_stack_variable_aligned = tbb::internal::is_aligned(&x,sizeof(T)); + bool is_member_variable_aligned = tbb::internal::is_aligned(&ac.i,sizeof(T)); + bool is_struct_size_correct = (sizeof(AlignmentChecker)==2*sizeof(tbb::atomic)); + bool known_issue_condition = __TBB_FORCE_64BIT_ALIGNMENT_BROKEN && ( sizeof(T)==8); + //TODO: replace these ifs with KNOWN_ISSUE macro when it available + if (!is_stack_variable_aligned){ + std::string msg = "Compiler failed to properly align local atomic variable?; size:"+to_string(sizeof(T)) + " type: " + +to_string(name) + " location:" + to_string(&x) +"\n"; + if (known_issue_condition) { + REPORT(("Known issue: "+ msg).c_str()); + }else{ + ASSERT(false,msg.c_str()); + } + } + if (!is_member_variable_aligned){ + std::string msg = "Compiler failed to properly align atomic member variable?; size:"+to_string(sizeof(T)) + " type: " + +to_string(name) + " location:" + to_string(&ac.i) +"\n"; + if (known_issue_condition) { + REPORT(("Known issue: "+ msg).c_str()); + }else{ + ASSERT(false,msg.c_str()); + } + } + if (!is_struct_size_correct){ + std::string msg = "Compiler failed to properly add padding to structure with atomic member variable?; Structure size:"+to_string(sizeof(AlignmentChecker)) + + " atomic size:"+to_string(sizeof(tbb::atomic)) + " type: " + to_string(name) +"\n"; + if (known_issue_condition) { + REPORT(("Known issue: "+ msg).c_str()); + }else{ + ASSERT(false,msg.c_str()); + } + } + + AlignmentChecker array[5]; + for( int k=0; k<5; ++k ) { + bool is_member_variable_in_array_aligned = tbb::internal::is_aligned(&array[k].i,sizeof(T)); + if (!is_member_variable_in_array_aligned) { + std::string msg = "Compiler failed to properly align atomic member variable inside an array?; size:"+to_string(sizeof(T)) + " type:"+to_string(name) + + " location:" + to_string(&array[k].i) + "\n"; + if (known_issue_condition){ + REPORT(("Known issue: "+ msg).c_str()); + }else{ + ASSERT(false,msg.c_str()); + } + } + } +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning( disable: 4146 ) // unary minus operator applied to unsigned type, result still unsigned + #pragma warning( disable: 4334 ) // result of 32-bit shift implicitly converted to 64 bits +#endif + +/** T is an integral type. */ +template +void TestAtomicInteger( const char* name ) { + REMARK("testing atomic<%s> (size=%d)\n",name,sizeof(tbb::atomic)); + TestAlignment(name); + TestOperations(0L, T(-T(1)), T(1)); + for( int k=0; k(T(p), T(~(p)), T(1-(p))); + TestOperations(T(-(p)), T(~(-(p))), T(1-(-(p)))); + TestFetchAndAdd(T(-(p))); + } + TestParallel( name ); +} + +namespace test_indirection_helpers { + template + struct Foo { + T x, y, z; + }; +} + +template +void TestIndirection() { + using test_indirection_helpers::Foo; + Foo item; + tbb::atomic*> pointer; + pointer = &item; + for( int k=-10; k<=10; ++k ) { + // Test various syntaxes for indirection to fields with non-zero offset. + T value1=T(), value2=T(); + for( size_t j=0; jy = value1; + (*pointer).z = value2; + T result1 = (*pointer).y; + T result2 = pointer->z; + ASSERT( memcmp(&value1,&result1,sizeof(T))==0, NULL ); + ASSERT( memcmp(&value2,&result2,sizeof(T))==0, NULL ); + } + #if __TBB_ICC_BUILTIN_ATOMICS_POINTER_ALIASING_BROKEN + //prevent ICC compiler from assuming 'item' is unused and reusing it's storage + item.x = item.y=item.z; + #endif +} + +//! Test atomic +template +void TestAtomicPointer() { + REMARK("testing atomic pointer (%d)\n",int(sizeof(T))); + T array[1000]; + TestOperations(&array[500],&array[250],&array[750]); + TestFetchAndAdd(&array[500]); + TestIndirection(); + TestParallel( "pointer" ); + +} + +//! Test atomic where Ptr is a pointer to a type of unknown size +template +void TestAtomicPointerToTypeOfUnknownSize( const char* name ) { + REMARK("testing atomic<%s>\n",name); + char array[1000]; + TestOperations((Ptr)(void*)&array[500],(Ptr)(void*)&array[250],(Ptr)(void*)&array[750]); + TestParallel( name ); +} + +void TestAtomicBool() { + REMARK("testing atomic\n"); + TestOperations(false,true,true); + TestOperations(true,false,false); + TestParallel( "bool" ); +} + +template +struct HasImplicitConversionToInt { + typedef bool yes; + typedef int no; + __TBB_STATIC_ASSERT( sizeof(yes) != sizeof(no), "The helper needs two types of different sizes to work." ); + + static yes detect( int ); + static no detect( ... ); + + enum { value = (sizeof(yes) == sizeof(detect( EnumType() ))) }; +}; + +enum Color {Red=0,Green=1,Blue=-1}; + +void TestAtomicEnum() { + REMARK("testing atomic\n"); + TestOperations(Red,Green,Blue); + TestParallel( "Color" ); + __TBB_STATIC_ASSERT( HasImplicitConversionToInt< tbb::atomic >::value, "The implicit conversion is expected." ); +} + +#if __TBB_SCOPED_ENUM_PRESENT +enum class ScopedColor1 {ScopedRed,ScopedGreen,ScopedBlue=-1}; +// TODO: extend the test to cover 2 byte scoped enum as well +#if __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_NEGATIVE_VALUE_BROKEN +enum class ScopedColor2 : signed char {ScopedZero, ScopedOne,ScopedRed=42,ScopedGreen=-1,ScopedBlue=127}; +#else +enum class ScopedColor2 : signed char {ScopedZero, ScopedOne,ScopedRed=-128,ScopedGreen=-1,ScopedBlue=127}; +#endif + +// TODO: replace the hack of getting symbolic enum name with a better implementation +std::string enum_strings[] = {"ScopedZero","ScopedOne","ScopedRed","ScopedGreen","ScopedBlue"}; +template<> +std::string to_string(const ScopedColor1& a){ + return enum_strings[a==ScopedColor1::ScopedBlue? 4 : (int)a+2]; +} +template<> +std::string to_string(const ScopedColor2& a){ + return enum_strings[a==ScopedColor2::ScopedRed? 2 : + a==ScopedColor2::ScopedGreen? 3 : a==ScopedColor2::ScopedBlue? 4 : (int)a ]; +} + +void TestAtomicScopedEnum() { + REMARK("testing atomic\n"); + TestOperations(ScopedColor1::ScopedRed,ScopedColor1::ScopedGreen,ScopedColor1::ScopedBlue); + TestParallel( "ScopedColor1" ); +#if __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_ATOMIC_LOAD_BROKEN + REPORT("Known issue: the operation tests for a scoped enum with a specified underlying type are skipped.\n"); +#else + TestOperations(ScopedColor2::ScopedRed,ScopedColor2::ScopedGreen,ScopedColor2::ScopedBlue); + TestParallel( "ScopedColor2" ); +#endif + __TBB_STATIC_ASSERT( !HasImplicitConversionToInt< tbb::atomic >::value, "The implicit conversion is not expected." ); + __TBB_STATIC_ASSERT( !HasImplicitConversionToInt< tbb::atomic >::value, "The implicit conversion is not expected." ); + __TBB_STATIC_ASSERT( sizeof(tbb::atomic) == sizeof(ScopedColor1), "tbb::atomic instantiated with scoped enum should have the same size as scoped enum." ); + __TBB_STATIC_ASSERT( sizeof(tbb::atomic) == sizeof(ScopedColor2), "tbb::atomic instantiated with scoped enum should have the same size as scoped enum." ); +} +#endif /* __TBB_SCOPED_ENUM_PRESENT */ + +template +void TestAtomicFloat( const char* name ) { + REMARK("testing atomic<%s>\n", name ); + TestAlignment(name); + TestOperations(0.5,3.25,10.75); + TestParallel( name ); +} + +#define __TBB_TEST_GENERIC_PART_WORD_CAS (__TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED) +#if __TBB_TEST_GENERIC_PART_WORD_CAS +void TestEndianness() { + // Test for pure endianness (assumed by simpler probe in __TBB_MaskedCompareAndSwap()). + bool is_big_endian = true, is_little_endian = true; + const tbb::internal::uint32_t probe = 0x03020100; + ASSERT (tbb::internal::is_aligned(&probe,4), NULL); + for( const char *pc_begin = reinterpret_cast(&probe) + , *pc = pc_begin, *pc_end = pc_begin + sizeof(probe) + ; pc != pc_end; ++pc) { + if (*pc != pc_end-1-pc) is_big_endian = false; + if (*pc != pc-pc_begin) is_little_endian = false; + } + ASSERT (!is_big_endian || !is_little_endian, NULL); + #if __TBB_ENDIANNESS==__TBB_ENDIAN_DETECT + ASSERT (is_big_endian || is_little_endian, "__TBB_ENDIANNESS should be set to __TBB_ENDIAN_UNSUPPORTED"); + #elif __TBB_ENDIANNESS==__TBB_ENDIAN_BIG + ASSERT (is_big_endian, "__TBB_ENDIANNESS should NOT be set to __TBB_ENDIAN_BIG"); + #elif __TBB_ENDIANNESS==__TBB_ENDIAN_LITTLE + ASSERT (is_little_endian, "__TBB_ENDIANNESS should NOT be set to __TBB_ENDIAN_LITTLE"); + #elif __TBB_ENDIANNESS==__TBB_ENDIAN_UNSUPPORTED + #error Generic implementation of part-word CAS may not be used: unsupported endianness + #else + #error Unexpected value of __TBB_ENDIANNESS + #endif +} + +namespace masked_cas_helpers { + const int numMaskedOperations = 100000; + const int testSpaceSize = 8; + int prime[testSpaceSize] = {3,5,7,11,13,17,19,23}; + + template + class TestMaskedCAS_Body: NoAssign { + T* test_space_uncontended; + T* test_space_contended; + public: + TestMaskedCAS_Body( T* _space1, T* _space2 ) : test_space_uncontended(_space1), test_space_contended(_space2) {} + void operator()( int my_idx ) const { + using tbb::internal::__TBB_MaskedCompareAndSwap; + const volatile T my_prime = T(prime[my_idx]); // 'volatile' prevents erroneous optimizations by SunCC + T* const my_ptr = test_space_uncontended+my_idx; + T old_value=0; + for( int i=0; i(my_ptr,new_value,old_value-1); + ASSERT(result!=old_value-1, "masked CAS succeeded while it should fail"); + ASSERT(result==*my_ptr, "masked CAS result mismatch with real value"); + // The following one should succeed + result = __TBB_MaskedCompareAndSwap(my_ptr,new_value,old_value); + ASSERT(result==old_value && *my_ptr==new_value, "masked CAS failed while it should succeed"); + // The following one should fail again + result = __TBB_MaskedCompareAndSwap(my_ptr,new_value,old_value); + ASSERT(result!=old_value, "masked CAS succeeded while it should fail"); + ASSERT(result==*my_ptr, "masked CAS result mismatch with real value"); + // Test contended case + for( int j=0; j(test_space_contended+j,value+my_prime,value); + } while( result!=value ); + } + } + } + }; + + template + struct intptr_as_array_of + { + static const int how_many_Ts = sizeof(intptr_t)/sizeof(T); + union { + intptr_t result; + T space[ how_many_Ts ]; + }; + }; + + template + intptr_t getCorrectUncontendedValue(int slot_idx) { + intptr_as_array_of slot; + slot.result = 0; + for( int i=0; i + intptr_t getCorrectContendedValue() { + intptr_as_array_of slot; + slot.result = 0; + for( int i=0; i +void TestMaskedCAS() { + using namespace masked_cas_helpers; + REMARK("testing masked CAS<%d>\n",int(sizeof(T))); + + const int num_slots = sizeof(T)*testSpaceSize/sizeof(intptr_t); + intptr_t arr1[num_slots+2]; // two more "canary" slots at boundaries + intptr_t arr2[num_slots+2]; + for(int i=0; i(test_space_uncontended, test_space_contended) ); + + ASSERT( arr1[0]==0 && arr1[num_slots+1]==0 && arr2[0]==0 && arr2[num_slots+1]==0 , "adjacent memory was overwritten" ); + const intptr_t correctContendedValue = getCorrectContendedValue(); + for(int i=0; i(i), "unexpected value in an uncontended slot" ); + ASSERT( arr2[i+1]==correctContendedValue, "unexpected value in a contended slot" ); + } +} +#endif // __TBB_TEST_GENERIC_PART_WORD_CAS + +template +class TestRelaxedLoadStorePlainBody { + static T s_turn, + s_ready; + +public: + static unsigned s_count1, + s_count2; + + void operator() ( int id ) const { + using tbb::internal::__TBB_load_relaxed; + using tbb::internal::__TBB_store_relaxed; + + if ( id == 0 ) { + while ( !__TBB_load_relaxed(s_turn) ) { + ++s_count1; + __TBB_store_relaxed(s_ready, 1); + } + } + else { + while ( !__TBB_load_relaxed(s_ready) ) { + ++s_count2; + continue; + } + __TBB_store_relaxed(s_turn, 1); + } + } +}; // class TestRelaxedLoadStorePlainBody + +template T TestRelaxedLoadStorePlainBody::s_turn = 0; +template T TestRelaxedLoadStorePlainBody::s_ready = 0; +template unsigned TestRelaxedLoadStorePlainBody::s_count1 = 0; +template unsigned TestRelaxedLoadStorePlainBody::s_count2 = 0; + +template +class TestRelaxedLoadStoreAtomicBody { + static tbb::atomic s_turn, + s_ready; + +public: + static unsigned s_count1, + s_count2; + + void operator() ( int id ) const { + if ( id == 0 ) { + while ( s_turn.template load() == 0 ) { + ++s_count1; + s_ready.template store(1); + } + } + else { + while ( s_ready.template load() == 0 ) { + ++s_count2; + continue; + } + s_turn.template store(1); + } + } +}; // class TestRelaxedLoadStoreAtomicBody + +template tbb::atomic TestRelaxedLoadStoreAtomicBody::s_turn; +template tbb::atomic TestRelaxedLoadStoreAtomicBody::s_ready; +template unsigned TestRelaxedLoadStoreAtomicBody::s_count1 = 0; +template unsigned TestRelaxedLoadStoreAtomicBody::s_count2 = 0; + +template +void TestRegisterPromotionSuppression () { + REMARK("testing register promotion suppression (size=%d)\n", (int)sizeof(T)); + NativeParallelFor( 2, TestRelaxedLoadStorePlainBody() ); + NativeParallelFor( 2, TestRelaxedLoadStoreAtomicBody() ); +} + +template +class ArrayElement { + char item[N]; +}; + +#include "harness_barrier.h" +namespace bit_operation_test_suite{ + struct fixture : NoAssign{ + static const uintptr_t zero = 0; + const uintptr_t random_value ; + const uintptr_t inverted_random_value ; + fixture(): + random_value (tbb::internal::select_size_t_constant<0x9E3779B9,0x9E3779B97F4A7C15ULL>::value), + inverted_random_value ( ~random_value) + {} + }; + + struct TestAtomicORSerially : fixture { + void operator()(){ + //these additional variable are needed to get more meaningful expression in the assert + uintptr_t initial_value = zero; + uintptr_t atomic_or_result = initial_value; + uintptr_t atomic_or_operand = random_value; + + __TBB_AtomicOR(&atomic_or_result,atomic_or_operand); + + ASSERT(atomic_or_result == (initial_value | atomic_or_operand),"AtomicOR should do the OR operation"); + } + }; + struct TestAtomicANDSerially : fixture { + void operator()(){ + //these additional variable are needed to get more meaningful expression in the assert + uintptr_t initial_value = inverted_random_value; + uintptr_t atomic_and_result = initial_value; + uintptr_t atomic_and_operand = random_value; + + __TBB_AtomicAND(&atomic_and_result,atomic_and_operand); + + ASSERT(atomic_and_result == (initial_value & atomic_and_operand),"AtomicAND should do the AND operation"); + } + }; + + struct TestAtomicORandANDConcurrently : fixture { + static const uintptr_t bit_per_word = sizeof(uintptr_t) * 8; + static const uintptr_t threads_number = bit_per_word; + Harness::SpinBarrier m_barrier; + uintptr_t bitmap; + TestAtomicORandANDConcurrently():bitmap(zero) {} + + struct thread_body{ + TestAtomicORandANDConcurrently* test; + thread_body(TestAtomicORandANDConcurrently* the_test) : test(the_test) {} + void operator()(int thread_index)const{ + const uintptr_t single_bit_mask = ((uintptr_t)1u) << (thread_index % bit_per_word); + test->m_barrier.wait(); + static const char* error_msg = "AtomicOR and AtomicAND should be atomic"; + for (uintptr_t attempts=0; attempts<1000; attempts++ ){ + //Set and clear designated bits in a word. + __TBB_AtomicOR(&test->bitmap,single_bit_mask); + __TBB_Yield(); + bool the_bit_is_set_after_set_via_atomic_or = ((__TBB_load_with_acquire(test->bitmap) & single_bit_mask )== single_bit_mask); + ASSERT(the_bit_is_set_after_set_via_atomic_or,error_msg); + + __TBB_AtomicAND(&test->bitmap,~single_bit_mask); + __TBB_Yield(); + bool the_bit_is_clear_after_clear_via_atomic_and = ((__TBB_load_with_acquire(test->bitmap) & single_bit_mask )== zero); + ASSERT(the_bit_is_clear_after_clear_via_atomic_and,error_msg); + } + } + }; + void operator()(){ + m_barrier.initialize(threads_number); + NativeParallelFor(threads_number,thread_body(this)); + } + }; +} +void TestBitOperations(){ + using namespace bit_operation_test_suite; + TestAtomicORSerially()(); + TestAtomicANDSerially()(); + TestAtomicORandANDConcurrently()(); +} + +int TestMain () { +# if __TBB_ATOMIC_CTORS + TestConstExprInitializationOfGlobalObjects(); +# endif //__TBB_ATOMIC_CTORS +# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN + TestAtomicInteger("unsigned long long"); + TestAtomicInteger("long long"); +# elif __TBB_CAS_8_CODEGEN_BROKEN + REPORT("Known issue: compiler generates incorrect code for 64-bit atomics on this configuration\n"); +# else + REPORT("Known issue: 64-bit atomics are not supported\n"); + ASSERT(sizeof(long long)==8, "type long long is not 64 bits"); +# endif + TestAtomicInteger("unsigned long"); + TestAtomicInteger("long"); + TestAtomicInteger("unsigned int"); + TestAtomicInteger("int"); + TestAtomicInteger("unsigned short"); + TestAtomicInteger("short"); + TestAtomicInteger("signed char"); + TestAtomicInteger("unsigned char"); + TestAtomicInteger("char"); + TestAtomicInteger("wchar_t"); + TestAtomicInteger("size_t"); + TestAtomicInteger("ptrdiff_t"); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointer >(); + TestAtomicPointerToTypeOfUnknownSize( "IncompleteType*" ); + TestAtomicPointerToTypeOfUnknownSize( "void*" ); + TestAtomicBool(); + TestAtomicEnum(); +# if __TBB_SCOPED_ENUM_PRESENT + TestAtomicScopedEnum(); +# endif + TestAtomicFloat("float"); +# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN + TestAtomicFloat("double"); +# else + ASSERT(sizeof(double)==8, "type double is not 64 bits"); +# endif + ASSERT( !ParallelError, NULL ); +# if __TBB_TEST_GENERIC_PART_WORD_CAS + TestEndianness(); + ASSERT (sizeof(short)==2, NULL); + TestMaskedCAS(); + TestMaskedCAS(); + TestMaskedCAS(); + TestMaskedCAS(); + TestMaskedCAS(); +# elif __TBB_USE_GENERIC_PART_WORD_CAS +# error Generic part-word CAS is enabled, but not covered by the test +# else + REPORT("Skipping test for generic part-word CAS\n"); +# endif +# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN + TestRegisterPromotionSuppression(); +# endif + TestRegisterPromotionSuppression(); + TestRegisterPromotionSuppression(); + TestRegisterPromotionSuppression(); + TestBitOperations(); + + return Harness::Done; +} + +template +class AlignedAtomic: NoAssign { + //tbb::aligned_space can not be used here, because internally it utilize align pragma/attribute, + //which has bugs on 8byte alignment on ia32 on some compilers( see according ****_BROKEN macro) + // Allocate space big enough to always contain sizeof(T)-byte locations that are aligned and misaligned. + char raw_space[2*sizeof(T) -1]; +public: + tbb::atomic& construct_atomic(){ + std::memset(&raw_space[0],0, sizeof(raw_space)); + uintptr_t delta = aligned ? 0 : sizeof(T)/2; + size_t index=sizeof(T)-1; + tbb::atomic* y = reinterpret_cast*>((reinterpret_cast(&raw_space[index+delta])&~index) - delta); + // Assertion checks that y really did end up somewhere inside "raw_space". + ASSERT( raw_space<=reinterpret_cast(y), "y starts before raw_space" ); + ASSERT( reinterpret_cast(y+1) <= raw_space+sizeof(raw_space), "y starts after raw_space" ); + ASSERT( !(aligned ^ tbb::internal::is_aligned(y,sizeof(T))), "y is not aligned as it required" ); + return *(new (y) tbb::atomic()); + } +}; + +template +struct FlagAndMessage: AlignedAtomic { + //! 0 if message not set yet, 1 if message is set. + tbb::atomic& flag; + /** Force flag and message to be on distinct cache lines for machines with cache line size <= 4096 bytes */ + char pad[4096/sizeof(T)]; + //! Non-zero if message is ready + T message; + FlagAndMessage(): flag(FlagAndMessage::construct_atomic()) { + std::memset(pad,0,sizeof(pad)); + } +}; + +// A special template function used for summation. +// Actually it is only necessary because of its specialization for void* +template +T special_sum(intptr_t arg1, intptr_t arg2) { + return (T)((T)arg1 + arg2); +} + +// The specialization for IncompleteType* is required +// because pointer arithmetic (+) is impossible with IncompleteType* +template<> +IncompleteType* special_sum(intptr_t arg1, intptr_t arg2) { + return (IncompleteType*)(arg1 + arg2); +} + +// The specialization for void* is required +// because pointer arithmetic (+) is impossible with void* +template<> +void* special_sum(intptr_t arg1, intptr_t arg2) { + return (void*)(arg1 + arg2); +} + +// The specialization for bool is required to shut up gratuitous compiler warnings, +// because some compilers warn about casting int to bool. +template<> +bool special_sum(intptr_t arg1, intptr_t arg2) { + return ((arg1!=0) + arg2)!=0; +} + +#if __TBB_SCOPED_ENUM_PRESENT +// The specialization for scoped enumerators is required +// because scoped enumerators prohibit implicit conversion to int +template<> +ScopedColor1 special_sum(intptr_t arg1, intptr_t arg2) { + return (ScopedColor1)(arg1 + arg2); +} +template<> +ScopedColor2 special_sum(intptr_t arg1, intptr_t arg2) { + return (ScopedColor2)(arg1 + arg2); +} +#endif + +volatile int One = 1; + +inline bool IsRelaxed ( LoadStoreExpression e ) { + return e == UseExplicitRelaxed || e == UseGlobalHelperRelaxed; +} + +template +struct LoadStoreTraits; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = src; } + static void store ( tbb::atomic& dst, const T& src ) { dst = src; } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = src.load(); } + static void store ( tbb::atomic& dst, const T& src ) { dst.store(src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = src.template load(); } + static void store ( tbb::atomic& dst, const T& src ) { dst.template store(src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = src.template load(); } + static void store ( tbb::atomic& dst, const T& src ) { dst.template store(src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = src.template load(); } + static void store ( tbb::atomic& dst, const T& src ) { dst.template store(src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = tbb::load(src); } + static void store ( tbb::atomic& dst, const T& src ) { tbb::store(dst, src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = tbb::load(src); } + static void store ( tbb::atomic& dst, const T& src ) { tbb::store(dst, src); } +}; + +template +struct LoadStoreTraits { + static void load ( T& dst, const tbb::atomic& src ) { dst = tbb::load(src); } + static void store ( tbb::atomic& dst, const T& src ) { tbb::store(dst, src); } +}; + +template +struct HammerLoadAndStoreFence: NoAssign { + typedef FlagAndMessage fam_type; +private: + typedef LoadStoreTraits trait; + fam_type* fam; + const int n; + const int p; + const int trial; + const char* name; + mutable T accum; +public: + HammerLoadAndStoreFence( fam_type* fam_, int n_, int p_, const char* name_, int trial_ ) : fam(fam_), n(n_), p(p_), trial(trial_), name(name_) {} + void operator()( int k ) const { + int one = One; + fam_type* s = fam+k; + fam_type* s_next = fam + (k+1)%p; + for( int i=0; iflag ); + message = s->message; + } else { + trait::load( flag, s->flag ); + message = s->message; + } + if ( flag != T(0) ) { + if( flag!=(T)-1 ) { + REPORT("ERROR: flag!=(T)-1 k=%d i=%d trial=%x type=%s (atomicity problem?)\n", k, i, trial, name ); + ParallelError = true; + } + if( !IsRelaxed(E) && message!=(T)-1 ) { + REPORT("ERROR: message!=(T)-1 k=%d i=%d trial=%x type=%s mode=%d (memory fence problem?)\n", k, i, trial, name, E ); + ParallelError = true; + } + s->message = T(0); + trait::store( s->flag, T(0) ); + // Prevent deadlock possible in relaxed mode because of store(0) + // to the first thread's flag being reordered after the last + // thread's store(-1) into it. + if ( IsRelaxed(E) ) { + while( s_next->flag.template load() != T(0) ) + __TBB_Yield(); + } + else + ASSERT( s_next->flag == T(0), NULL ); + // Set message and then the flag + if( trial&2 ) { + // COMPLICATED_ZERO here tempts compiler to sink store below setting of flag + s_next->message = special_sum(-1, COMPLICATED_ZERO); + trait::store( s_next->flag, (T)-1 ); + } else { + s_next->message = (T)-1; + trait::store( s_next->flag, (T)-1 ); + } + break; + } else { + // Force compiler to use message anyway, so it cannot sink read of s->message below the if. + accum = message; + } + } + } + } +}; + +//! Test that atomic has acquire semantics for loads and release semantics for stores. +/** Test performs round-robin passing of message among p processors, + where p goes from MinThread to MaxThread. */ +template +void TestLoadAndStoreFences( const char* name ) { + typedef HammerLoadAndStoreFence hammer_load_store_type; + typedef typename hammer_load_store_type::fam_type fam_type; + for( int p=MinThread<2 ? 2 : MinThread; p<=MaxThread; ++p ) { + fam_type * fam = new fam_type[p]; + // Each of four trials exercise slightly different expression pattern within the test. + // See occurrences of COMPLICATED_ZERO for details. + for( int trial=0; trial<4; ++trial ) { + fam->message = (T)-1; + fam->flag = (T)-1; + NativeParallelFor( p, hammer_load_store_type( fam, 100, p, name, trial ) ); + if ( !IsRelaxed(E) ) { + for( int k=0; k +class SparseValueSet { + T factor; +public: + SparseValueSet() { + // Compute factor such that: + // 1. It has at least one 1 in most of its bytes. + // 2. The bytes are typically different. + // 3. When multiplied by any value <=127, the product does not overflow. + factor = T(0); + for( unsigned i=0; i +class SparseValueSet { + SparseValueSet my_set; +public: + T* get( int i ) const {return reinterpret_cast(my_set.get(i));} + bool contains( T* x ) const {return my_set.contains(reinterpret_cast(x));} +}; + +//! Specialization for bool. +/** Checking bool for atomic read/write is pointless in practice, because + there is no way to *not* atomically read or write a bool value. */ +template<> +class SparseValueSet { +public: + bool get( int i ) const {return i&1;} + bool contains( bool ) const {return true;} +}; + +#if _MSC_VER==1500 && !defined(__INTEL_COMPILER) + // VS2008/VC9 seems to have an issue; limits pull in math.h + #pragma warning( push ) + #pragma warning( disable: 4985 ) +#endif +#include /* Need std::numeric_limits */ +#if _MSC_VER==1500 && !defined(__INTEL_COMPILER) + #pragma warning( pop ) +#endif + +//! Commonality inherited by specializations for floating-point types. +template +class SparseFloatSet: NoAssign { + const T epsilon; +public: + SparseFloatSet() : epsilon(std::numeric_limits::epsilon()) {} + T get( int i ) const { + return i==0 ? T(0) : 1/T((i&0x7F)+1); + } + bool contains( T x ) const { + if( x==T(0) ) { + return true; + } else { + int j = int(1/x+T(0.5)); + if( 0 +class SparseValueSet: public SparseFloatSet {}; + +template<> +class SparseValueSet: public SparseFloatSet {}; + +#if __TBB_SCOPED_ENUM_PRESENT +//! Commonality inherited by specializations for scoped enumerator types. +template +class SparseEnumValueSet { +public: + EnumType get( int i ) const {return i%3==0 ? EnumType::ScopedRed : i%3==1 ? EnumType::ScopedGreen : EnumType::ScopedBlue;} + bool contains( EnumType e ) const {return e==EnumType::ScopedRed || e==EnumType::ScopedGreen || e==EnumType::ScopedBlue;} +}; +template<> +class SparseValueSet : public SparseEnumValueSet {}; +template<> +class SparseValueSet : public SparseEnumValueSet {}; +#endif + +template +class HammerAssignment: AlignedAtomic { + tbb::atomic& x; + const char* name; + SparseValueSet set; +public: + HammerAssignment(const char* name_ ) : x(HammerAssignment::construct_atomic()), name(name_) { + x = set.get(0); + } + void operator()( int k ) const { + const int n = 1000000; + if( k ) { + tbb::atomic z; + AssertSameType( z=x, z ); // Check that return type from assignment is correct + for( int i=0; i is not atomic\n", name); + ParallelError = true; + return; + } + } + } else { + tbb::atomic y; + for( int i=0; i void TestAssignmentSignature( T& (T::*)(const T&) ) {} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning( disable: 4355 4800 ) +#endif + +template +void TestAssignment( const char* name ) { + TestAssignmentSignature( &tbb::atomic::operator= ); + NativeParallelFor( 2, HammerAssignment(name ) ); +} + +template +class DekkerArbitrationBody : NoAssign, Harness::NoAfterlife { + typedef LoadStoreTraits trait; + + mutable Harness::FastRandom my_rand; + static const unsigned short c_rand_ceil = 10; + mutable AlignedAtomic s_ready_storage[2]; + mutable AlignedAtomic s_turn_storage; + mutable tbb::atomic* s_ready[2]; + tbb::atomic& s_turn; + mutable volatile bool s_inside; + +public: + void operator() ( int id ) const { + const int me = id; + const T other = (T)(uintptr_t)(1 - id), + cleared = T(0), + signaled = T(1); + for ( int i = 0; i < 100000; ++i ) { + trait::store( *s_ready[me], signaled ); + trait::store( s_turn, other ); + T r, t; + for ( int j = 0; ; ++j ) { + trait::load(r, *s_ready[(uintptr_t)other]); + trait::load(t, s_turn); + if ( r != signaled || t != other ) + break; + __TBB_Pause(1); + if ( j == 2<<12 ) { + j = 0; + __TBB_Yield(); + } + } + // Entered critical section + ASSERT( !s_inside, "Peterson lock is broken - some fences are missing" ); + s_inside = true; + unsigned short spin = my_rand.get() % c_rand_ceil; + for ( volatile int j = 0; j < spin; ++j ) + continue; + s_inside = false; + ASSERT( !s_inside, "Peterson lock is broken - some fences are missing" ); + // leaving critical section + trait::store( *s_ready[me], cleared ); + spin = my_rand.get() % c_rand_ceil; + for ( volatile int j = 0; j < spin; ++j ) + continue; + } + } + + DekkerArbitrationBody () + : my_rand((unsigned)(uintptr_t)this) + , s_turn(s_turn_storage.construct_atomic()) + , s_inside (false) + { + //atomics pointed to by s_ready and s_turn will be zeroed by the + //according construct_atomic() calls + s_ready[0] = &s_ready_storage[0].construct_atomic(); + s_ready[1] = &s_ready_storage[1].construct_atomic(); + } +}; + +template +void TestDekkerArbitration () { + NativeParallelFor( 2, DekkerArbitrationBody() ); +} + +template +void TestParallel( const char* name ) { + //TODO: looks like there are no tests for operations other than load/store ? +#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN + if (sizeof(T)==8){ + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestAssignment(name); + TestDekkerArbitration(); + TestDekkerArbitration(); + } +#endif + + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestLoadAndStoreFences(name); + TestAssignment(name); + TestDekkerArbitration(); + TestDekkerArbitration(); +} + +#endif // __TBB_TEST_SKIP_PIC_MODE || __TBB_TEST_SKIP_BUILTINS_MODE diff --git a/src/tbb/src/test/test_blocked_range.cpp b/src/tbb/src/test/test_blocked_range.cpp new file mode 100644 index 00000000..245405d6 --- /dev/null +++ b/src/tbb/src/test/test_blocked_range.cpp @@ -0,0 +1,206 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/blocked_range.h" +#include "harness_assert.h" + +// First test as much as we can without including other headers. +// Doing so should catch problems arising from failing to include headers. + +class AbstractValueType { + AbstractValueType() {} + int value; +public: + friend AbstractValueType MakeAbstractValueType( int i ); + friend int GetValueOf( const AbstractValueType& v ) {return v.value;} +}; + +AbstractValueType MakeAbstractValueType( int i ) { + AbstractValueType x; + x.value = i; + return x; +} + +std::size_t operator-( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u) - GetValueOf(v); +} + +bool operator<( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u) < GetValueOf(v); +} + +AbstractValueType operator+( const AbstractValueType& u, std::size_t offset ) { + return MakeAbstractValueType(GetValueOf(u) + int(offset)); +} + +static void SerialTest() { + for( int x=-10; x<10; ++x ) + for( int y=-10; y<10; ++y ) { + AbstractValueType i = MakeAbstractValueType(x); + AbstractValueType j = MakeAbstractValueType(y); + for( std::size_t k=1; k<10; ++k ) { + typedef tbb::blocked_range range_type; + range_type r( i, j, k ); + AssertSameType( r.empty(), true ); + AssertSameType( range_type::size_type(), std::size_t() ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( r.begin(), MakeAbstractValueType(0) ); + AssertSameType( r.end(), MakeAbstractValueType(0) ); + ASSERT( r.empty()==(y<=x), NULL ); + ASSERT( r.grainsize()==k, NULL ); + if( x<=y ) { + AssertSameType( r.is_divisible(), true ); + ASSERT( r.is_divisible()==(std::size_t(y-x)>k), NULL ); + ASSERT( r.size()==std::size_t(y-x), NULL ); + if( r.is_divisible() ) { + tbb::blocked_range r2(r,tbb::split()); + ASSERT( GetValueOf(r.begin())==x, NULL ); + ASSERT( GetValueOf(r.end())==GetValueOf(r2.begin()), NULL ); + ASSERT( GetValueOf(r2.end())==y, NULL ); + ASSERT( r.grainsize()==k, NULL ); + ASSERT( r2.grainsize()==k, NULL ); + } + } + } + } +} + +#include "tbb/parallel_for.h" +#include "harness.h" + +const int N = 1<<22; + +unsigned char Array[N]; + +struct Striker { + // Note: we use here instead of in order to test for Quad 407676 + void operator()( const tbb::blocked_range& r ) const { + for( tbb::blocked_range::const_iterator i=r.begin(); i!=r.end(); ++i ) + ++Array[i]; + } +}; + +void ParallelTest() { + for( int i=0; i r( 0, i, 10 ); + tbb::parallel_for( r, Striker() ); + for( int k=0; k +void TestRangeBasedFor() { + using namespace range_based_for_support_tests; + REMARK("testing range based for loop compatibility \n"); + + size_t int_array[100] = {0}; + const size_t sequence_length = Harness::array_length(int_array); + + for (size_t i = 0; i < sequence_length; ++i) { + int_array[i] = i + 1; + } + + const tbb::blocked_range r(int_array, Harness::end(int_array), 1); + + ASSERT(range_based_for_accumulate(r, std::plus(), size_t(0)) == gauss_summ_of_int_sequence(sequence_length), "incorrect accumulated value generated via range based for ?"); +} +#endif //if __TBB_RANGE_BASED_FOR_PRESENT + +#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES + +void TestProportionalSplitOverflow() +{ + REMARK("Testing overflow during proportional split - "); + using tbb::blocked_range; + using tbb::proportional_split; + + blocked_range r1(0, size_t(-1) / 2); + size_t size = r1.size(); + size_t begin = r1.begin(); + size_t end = r1.end(); + + proportional_split p(1, 3); + blocked_range r2(r1, p); + + // overflow-free computation + size_t parts = p.left() + p.right(); + size_t int_part = size / parts; + size_t fraction = size - int_part * parts; // fraction < parts + size_t right_idx = int_part * p.right() + fraction * p.right() / parts + 1; + size_t newRangeBegin = end - right_idx; + + // Division in 'right_idx' very likely is inexact also. + size_t tolerance = 1; + size_t diff = (r2.begin() < newRangeBegin) ? (newRangeBegin - r2.begin()) : (r2.begin() - newRangeBegin); + bool is_split_correct = diff <= tolerance; + bool test_passed = (r1.begin() == begin && r1.end() == r2.begin() && is_split_correct && + r2.end() == end); + if (!test_passed) { + REPORT("Incorrect split of blocked range[%lu, %lu) into r1[%lu, %lu) and r2[%lu, %lu), " + "must be r1[%lu, %lu) and r2[%lu, %lu)\n", begin, end, r1.begin(), r1.end(), r2.begin(), r2.end(), begin, newRangeBegin, newRangeBegin, end); + ASSERT(test_passed, NULL); + } + REMARK("OK\n"); +} +#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +void TestDeductionGuides() { + std::vector v; + + // check blocked_range(Value, Value, size_t) + tbb::blocked_range r1(v.begin(), v.end()); + static_assert(std::is_same>::value); + + // check blocked_range(blocked_range &) + tbb::blocked_range r2(r1); + static_assert(std::is_same::value); + + // check blocked_range(blocked_range &&) + tbb::blocked_range r3(std::move(r1)); + static_assert(std::is_same::value); +} +#endif + +//------------------------------------------------------------------------ +// Test driver +#include "tbb/task_scheduler_init.h" + +int TestMain () { + SerialTest(); + for( int p=MinThread; p<=MaxThread; ++p ) { + tbb::task_scheduler_init init(p); + ParallelTest(); + } + + #if __TBB_RANGE_BASED_FOR_PRESENT + TestRangeBasedFor(); + #endif //if __TBB_RANGE_BASED_FOR_PRESENT + + #if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES + TestProportionalSplitOverflow(); + #endif + + #if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + TestDeductionGuides(); + #endif + return Harness::Done; +} diff --git a/src/tbb/src/test/test_blocked_range2d.cpp b/src/tbb/src/test/test_blocked_range2d.cpp new file mode 100644 index 00000000..306eedd4 --- /dev/null +++ b/src/tbb/src/test/test_blocked_range2d.cpp @@ -0,0 +1,168 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/blocked_range2d.h" +#include "harness_assert.h" + +// First test as much as we can without including other headers. +// Doing so should catch problems arising from failing to include headers. + +template +class AbstractValueType { + AbstractValueType() {} + int value; +public: + template + friend AbstractValueType MakeAbstractValueType( int i ); + + template + friend int GetValueOf( const AbstractValueType& v ) ; +}; + +template +AbstractValueType MakeAbstractValueType( int i ) { + AbstractValueType x; + x.value = i; + return x; +} + +template +int GetValueOf( const AbstractValueType& v ) {return v.value;} + +template +bool operator<( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u) +std::size_t operator-( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u)-GetValueOf(v); +} + +template +AbstractValueType operator+( const AbstractValueType& u, std::size_t offset ) { + return MakeAbstractValueType(GetValueOf(u)+int(offset)); +} + +struct RowTag {}; +struct ColTag {}; + +static void SerialTest() { + typedef AbstractValueType row_type; + typedef AbstractValueType col_type; + typedef tbb::blocked_range2d range_type; + for( int row_x=-10; row_x<10; ++row_x ) { + for( int row_y=row_x; row_y<10; ++row_y ) { + row_type row_i = MakeAbstractValueType(row_x); + row_type row_j = MakeAbstractValueType(row_y); + for( int row_grain=1; row_grain<10; ++row_grain ) { + for( int col_x=-10; col_x<10; ++col_x ) { + for( int col_y=col_x; col_y<10; ++col_y ) { + col_type col_i = MakeAbstractValueType(col_x); + col_type col_j = MakeAbstractValueType(col_y); + for( int col_grain=1; col_grain<10; ++col_grain ) { + range_type r( row_i, row_j, row_grain, col_i, col_j, col_grain ); + AssertSameType( r.is_divisible(), true ); + AssertSameType( r.empty(), true ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( r.rows(), tbb::blocked_range( row_i, row_j, 1 )); + AssertSameType( r.cols(), tbb::blocked_range( col_i, col_j, 1 )); + ASSERT( r.empty()==(row_x==row_y||col_x==col_y), NULL ); + ASSERT( r.is_divisible()==(row_y-row_x>row_grain||col_y-col_x>col_grain), NULL ); + if( r.is_divisible() ) { + range_type r2(r,tbb::split()); + if( GetValueOf(r2.rows().begin())==GetValueOf(r.rows().begin()) ) { + ASSERT( GetValueOf(r2.rows().end())==GetValueOf(r.rows().end()), NULL ); + ASSERT( GetValueOf(r2.cols().begin())==GetValueOf(r.cols().end()), NULL ); + } else { + ASSERT( GetValueOf(r2.cols().end())==GetValueOf(r.cols().end()), NULL ); + ASSERT( GetValueOf(r2.rows().begin())==GetValueOf(r.rows().end()), NULL ); + } + } + } + } + } + } + } + } +} + +#include "tbb/parallel_for.h" +#include "harness.h" + +const int N = 1<<10; + +unsigned char Array[N][N]; + +struct Striker { + // Note: we use here instead of in order to test for problems similar to Quad 407676 + void operator()( const tbb::blocked_range2d& r ) const { + for( tbb::blocked_range::const_iterator i=r.rows().begin(); i!=r.rows().end(); ++i ) + for( tbb::blocked_range::const_iterator j=r.cols().begin(); j!=r.cols().end(); ++j ) + ++Array[i][j]; + } +}; + +void ParallelTest() { + for( int i=0; i r( 0, i, 7, 0, j, 5 ); + tbb::parallel_for( r, Striker() ); + for( int k=0; k +void TestDeductionGuides() { + std::vector v; + std::vector v2; + + // check blocked_range2d(RowValue, RowValue, size_t, ColValue, ColValue, size_t) + tbb::blocked_range2d r1(v.begin(), v.end(), 2, v2.begin(), v2.end(), 2); + static_assert(std::is_same>::value); + + // check blocked_range2d(blocked_range2d &) + tbb::blocked_range2d r2(r1); + static_assert(std::is_same::value); + + // check blocked_range2d(blocked_range2d &&) + tbb::blocked_range2d r3(std::move(r1)); + static_assert(std::is_same::value); +} +#endif + +#include "tbb/task_scheduler_init.h" + +int TestMain () { + SerialTest(); + for( int p=MinThread; p<=MaxThread; ++p ) { + tbb::task_scheduler_init init(p); + ParallelTest(); + } + + #if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + TestDeductionGuides(); + #endif + return Harness::Done; +} diff --git a/src/tbb/src/test/test_blocked_range3d.cpp b/src/tbb/src/test/test_blocked_range3d.cpp new file mode 100644 index 00000000..1609cf40 --- /dev/null +++ b/src/tbb/src/test/test_blocked_range3d.cpp @@ -0,0 +1,201 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/blocked_range3d.h" +#include "harness_assert.h" + +// First test as much as we can without including other headers. +// Doing so should catch problems arising from failing to include headers. + +template +class AbstractValueType { + AbstractValueType() {} + int value; +public: + template + friend AbstractValueType MakeAbstractValueType( int i ); + + template + friend int GetValueOf( const AbstractValueType& v ) ; +}; + +template +AbstractValueType MakeAbstractValueType( int i ) { + AbstractValueType x; + x.value = i; + return x; +} + +template +int GetValueOf( const AbstractValueType& v ) {return v.value;} + +template +bool operator<( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u) +std::size_t operator-( const AbstractValueType& u, const AbstractValueType& v ) { + return GetValueOf(u)-GetValueOf(v); +} + +template +AbstractValueType operator+( const AbstractValueType& u, std::size_t offset ) { + return MakeAbstractValueType(GetValueOf(u)+int(offset)); +} + +struct PageTag {}; +struct RowTag {}; +struct ColTag {}; + +static void SerialTest() { + typedef AbstractValueType page_type; + typedef AbstractValueType row_type; + typedef AbstractValueType col_type; + typedef tbb::blocked_range3d range_type; + for( int page_x=-4; page_x<4; ++page_x ) { + for( int page_y=page_x; page_y<4; ++page_y ) { + page_type page_i = MakeAbstractValueType(page_x); + page_type page_j = MakeAbstractValueType(page_y); + for( int page_grain=1; page_grain<4; ++page_grain ) { + for( int row_x=-4; row_x<4; ++row_x ) { + for( int row_y=row_x; row_y<4; ++row_y ) { + row_type row_i = MakeAbstractValueType(row_x); + row_type row_j = MakeAbstractValueType(row_y); + for( int row_grain=1; row_grain<4; ++row_grain ) { + for( int col_x=-4; col_x<4; ++col_x ) { + for( int col_y=col_x; col_y<4; ++col_y ) { + col_type col_i = MakeAbstractValueType(col_x); + col_type col_j = MakeAbstractValueType(col_y); + for( int col_grain=1; col_grain<4; ++col_grain ) { + range_type r( page_i, page_j, page_grain, row_i, row_j, row_grain, col_i, col_j, col_grain ); + AssertSameType( r.is_divisible(), true ); + + AssertSameType( r.empty(), true ); + + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + + AssertSameType( r.pages(), tbb::blocked_range( page_i, page_j, 1 )); + AssertSameType( r.rows(), tbb::blocked_range( row_i, row_j, 1 )); + AssertSameType( r.cols(), tbb::blocked_range( col_i, col_j, 1 )); + + ASSERT( r.empty()==(page_x==page_y||row_x==row_y||col_x==col_y), NULL ); + + ASSERT( r.is_divisible()==(page_y-page_x>page_grain||row_y-row_x>row_grain||col_y-col_x>col_grain), NULL ); + + if( r.is_divisible() ) { + range_type r2(r,tbb::split()); + if( (GetValueOf(r2.pages().begin())==GetValueOf(r.pages().begin())) && (GetValueOf(r2.rows().begin())==GetValueOf(r.rows().begin())) ) { + ASSERT( GetValueOf(r2.pages().end())==GetValueOf(r.pages().end()), NULL ); + ASSERT( GetValueOf(r2.rows().end())==GetValueOf(r.rows().end()), NULL ); + ASSERT( GetValueOf(r2.cols().begin())==GetValueOf(r.cols().end()), NULL ); + } else { + if ( (GetValueOf(r2.pages().begin())==GetValueOf(r.pages().begin())) && (GetValueOf(r2.cols().begin())==GetValueOf(r.cols().begin())) ) { + ASSERT( GetValueOf(r2.pages().end())==GetValueOf(r.pages().end()), NULL ); + ASSERT( GetValueOf(r2.cols().end())==GetValueOf(r.cols().end()), NULL ); + ASSERT( GetValueOf(r2.rows().begin())==GetValueOf(r.rows().end()), NULL ); + } else { + ASSERT( GetValueOf(r2.rows().end())==GetValueOf(r.rows().end()), NULL ); + ASSERT( GetValueOf(r2.cols().end())==GetValueOf(r.cols().end()), NULL ); + ASSERT( GetValueOf(r2.pages().begin())==GetValueOf(r.pages().end()), NULL ); + } + } + } + } + } + } + } + } + } + } + } + } +} + +#include "tbb/parallel_for.h" +#include "harness.h" + +const int N = 1<<5; + +unsigned char Array[N][N][N]; + +struct Striker { + // Note: we use here instead of in order to test for problems similar to Quad 407676 + void operator()( const tbb::blocked_range3d& r ) const { + for( tbb::blocked_range::const_iterator i=r.pages().begin(); i!=r.pages().end(); ++i ) + for( tbb::blocked_range::const_iterator j=r.rows().begin(); j!=r.rows().end(); ++j ) + for( tbb::blocked_range::const_iterator k=r.cols().begin(); k!=r.cols().end(); ++k ) + ++Array[i][j][k]; + } +}; + +void ParallelTest() { + for( int i=0; i r( 0, i, 5, 0, j, 3, 0, k, 1 ); + tbb::parallel_for( r, Striker() ); + for( int l=0; l +void TestDeductionGuides() { + std::vector v; + std::vector v2; + std::vector> v3; + + // check blocked_range2d(PageValue, PageValue, size_t, RowValue, RowValue, size_t, ColValue, ColValue, size_t) + tbb::blocked_range3d r1(v.begin(), v.end(), 2, v2.begin(), v2.end(), 2, v3.begin(), v3.end(), 6); + static_assert(std::is_same>::value); + + // check blocked_range2d(blocked_range3d &) + tbb::blocked_range3d r2(r1); + static_assert(std::is_same::value); + + // check blocked_range2d(blocked_range3d &&) + tbb::blocked_range3d r3(std::move(r1)); + static_assert(std::is_same::value); +} +#endif + +#include "tbb/task_scheduler_init.h" + +int TestMain () { + SerialTest(); + for( int p=MinThread; p<=MaxThread; ++p ) { + tbb::task_scheduler_init init(p); + ParallelTest(); + } + + #if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + TestDeductionGuides(); + #endif + return Harness::Done; +} diff --git a/src/tbb/src/test/test_blocked_rangeNd.cpp b/src/tbb/src/test/test_blocked_rangeNd.cpp new file mode 100644 index 00000000..413f5298 --- /dev/null +++ b/src/tbb/src/test/test_blocked_rangeNd.cpp @@ -0,0 +1,255 @@ +/* + Copyright (c) 2017-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define TBB_PREVIEW_BLOCKED_RANGE_ND 1 +#include "tbb/blocked_rangeNd.h" + +#include "tbb/tbb_config.h" + +#if __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT +// AbstractValueType class represents Value concept's requirements in the most abstract way +class AbstractValueType { + int value; + AbstractValueType() {} +public: + friend AbstractValueType MakeAbstractValue(int i); + friend int GetValueOf(const AbstractValueType& v); +}; + +int GetValueOf(const AbstractValueType& v) { return v.value; } + +AbstractValueType MakeAbstractValue(int i) { + AbstractValueType x; + x.value = i; + return x; +} + +// operator- returns amount of elements of AbstractValueType between u and v +std::size_t operator-(const AbstractValueType& u, const AbstractValueType& v) { + return GetValueOf(u) - GetValueOf(v); +} + +bool operator<(const AbstractValueType& u, const AbstractValueType& v) { + return GetValueOf(u) < GetValueOf(v); +} + +AbstractValueType operator+(const AbstractValueType& u, std::size_t offset) { + return MakeAbstractValue(GetValueOf(u) + int(offset)); +} + +#include "harness_assert.h" +#include // std::for_each +#include + +namespace internal { + template + struct utils { + using val_t = typename range_t::value_type; + + template + using data_type = std::array::template data_type, DimSize>; + + template + static void init_data(data_type& data) { + std::for_each(data.begin(), data.end(), utils::template init_data); + } + + template + static void increment_data(const range_t& range, data_type& data) { + auto begin = data.begin() + range.dim(N - 1).begin(); + // same as "auto end = out.begin() + range.dim(N - 1).end();" + auto end = begin + range.dim(N - 1).size(); + for (auto i = begin; i != end; ++i) { + utils::template increment_data(range, *i); + } + } + + template + static void check_data(const range_t& range, data_type& data) { + auto begin = data.begin() + range.dim(N - 1).begin(); + // same as "auto end = out.begin() + range.dim(N - 1).end();" + auto end = begin + range.dim(N - 1).size(); + for (auto i = begin; i != end; ++i) { + utils::template check_data(range, *i); + } + } + + template + static range_t make_range(std::size_t shift, bool negative, val_t(*gen)(input_t), tbb::internal::index_sequence) { + return range_t( { { + /* begin =*/gen(negative ? -input_t(Is + shift) : 0), + /* end =*/gen(input_t(Is + shift)), + /*grainsize =*/Is + 1} + /*pack expansion*/... } ); + } + + static bool is_empty(const range_t& range) { + if (range.dim(N - 1).empty()) { return true; } + return utils::is_empty(range); + } + + static bool is_divisible(const range_t& range) { + if (range.dim(N - 1).is_divisible()) { return true; } + return utils::is_divisible(range); + } + + static void check_splitting(const range_t& range_split, const range_t& range_new, int(*get)(const val_t&), bool split_checker = false) { + if (get(range_split.dim(N - 1).begin()) == get(range_new.dim(N - 1).begin())) { + ASSERT(get(range_split.dim(N - 1).end()) == get(range_new.dim(N - 1).end()), NULL); + } + else { + ASSERT(get(range_split.dim(N - 1).end()) == get(range_new.dim(N - 1).begin()) && !split_checker, NULL); + split_checker = true; + } + utils::check_splitting(range_split, range_new, get, split_checker); + } + + }; + + template + struct utils { + using val_t = typename range_t::value_type; + + template + using data_type = EntityType; + + template + static void init_data(data_type& data) { data = 0; } + + template + static void increment_data(const range_t&, data_type& data) { ++data; } + + template + static void check_data(const range_t&, data_type& data) { + ASSERT(data == 1, NULL); + } + + static bool is_empty(const range_t&) { return false; } + + static bool is_divisible(const range_t&) { return false; } + + static void check_splitting(const range_t&, const range_t&, int(*)(const val_t&), bool) {} + }; + + // We need MakeInt function to pass it into make_range as factory function + // because of matching make_range with AbstractValueType and other types too + int MakeInt(int i) { return i; } +} + +template +void SerialTest() { + __TBB_STATIC_ASSERT((tbb::blocked_rangeNd::ndims() + == tbb::blocked_rangeNd::ndims()), + "different amount of dimensions"); + + using range_t = tbb::blocked_rangeNd; + // 'typedef' instead of 'using' because of GCC 4.7.2 bug on Debian 7.0 + typedef internal::utils utils; + + // Generate empty range + range_t r = utils::make_range(0, true, &MakeAbstractValue, tbb::internal::make_index_sequence()); + + AssertSameType(r.is_divisible(), bool()); + AssertSameType(r.empty(), bool()); + AssertSameType(range_t::ndims(), 0U); + + ASSERT(r.empty() == utils::is_empty(r) && r.empty(), NULL); + ASSERT(r.is_divisible() == utils::is_divisible(r), NULL); + + // Generate not-empty range divisible range + r = utils::make_range(1, true, &MakeAbstractValue, tbb::internal::make_index_sequence()); + ASSERT(r.empty() == utils::is_empty(r) && !r.empty(), NULL); + ASSERT(r.is_divisible() == utils::is_divisible(r) && r.is_divisible(), NULL); + + range_t r_new(r, tbb::split()); + utils::check_splitting(r, r_new, &GetValueOf); + + SerialTest(); +} +template<> void SerialTest<0>() {} + +#include "tbb/parallel_for.h" + +template +void ParallelTest() { + using range_t = tbb::blocked_rangeNd; + // 'typedef' instead of 'using' because of GCC 4.7.2 bug on Debian 7.0 + typedef internal::utils utils; + + // Max size is 1 << 20 - 1 bytes + // Thus size of one dimension's elements is 1 << (20 / DimAmount - 1) bytes + typename utils::template data_type data; + utils::init_data(data); + + range_t r = utils::make_range((1 << (20 / DimAmount - 1)) - DimAmount, false, &internal::MakeInt, tbb::internal::make_index_sequence()); + + tbb::parallel_for(r, [&data](const range_t& range) { + utils::increment_data(range, data); + }); + + utils::check_data(r, data); + + ParallelTest(); +} +template<> void ParallelTest<0>() {} + +void TestCtors() { + tbb::blocked_rangeNd{ { 0,13,3 } }; + + tbb::blocked_rangeNd{ tbb::blocked_range{ 0,13,3 } }; + + tbb::blocked_rangeNd(tbb::blocked_range(-8923, 8884, 13), tbb::blocked_range(-8923, 5, 13)); + + tbb::blocked_rangeNd({ -8923, 8884, 13 }, { -8923, 8884, 13 }); + + tbb::blocked_range r1(0, 13); + + tbb::blocked_range r2(-12, 23); + + tbb::blocked_rangeNd({ { -8923, 8884, 13 }, r1}); + + tbb::blocked_rangeNd({ r2, r1 }); + + tbb::blocked_rangeNd(r1, r2); + + tbb::blocked_rangeNd({ MakeAbstractValue(-3), MakeAbstractValue(13), 8 }, + { MakeAbstractValue(-53), MakeAbstractValue(23), 2 }, + { MakeAbstractValue(-23), MakeAbstractValue(33), 1 }, + { MakeAbstractValue(-13), MakeAbstractValue(43), 7 }); +} + +static const std::size_t N = 4; + +#include "harness.h" +#include "tbb/task_scheduler_init.h" + +int TestMain() { + TestCtors(); + SerialTest(); + for( int p=MinThread; p<= MaxThread; ++p ) { + tbb::task_scheduler_init init(p); + ParallelTest(); + } + return Harness::Done; +} + +#else + +// tbb::blocked_rangeNd requires C++11 support +#define HARNESS_SKIP_TEST 1 +#include "harness.h" + +#endif /* __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ diff --git a/src/tbb/src/test/test_broadcast_node.cpp b/src/tbb/src/test/test_broadcast_node.cpp new file mode 100644 index 00000000..362419b2 --- /dev/null +++ b/src/tbb/src/test/test_broadcast_node.cpp @@ -0,0 +1,342 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "harness.h" + +#if __TBB_CPF_BUILD +#define TBB_DEPRECATED_FLOW_NODE_EXTRACTION 1 +#endif + +#include "tbb/flow_graph.h" +#include "tbb/task.h" +#include "tbb/atomic.h" + +const int N = 1000; +const int R = 4; + +class int_convertable_type : private NoAssign { + + int my_value; + +public: + + int_convertable_type( int v ) : my_value(v) {} + operator int() const { return my_value; } + +}; + + +template< typename T > +class counting_array_receiver : public tbb::flow::receiver { + + tbb::atomic my_counters[N]; + tbb::flow::graph& my_graph; + +public: + + counting_array_receiver(tbb::flow::graph& g) : my_graph(g) { + for (int i = 0; i < N; ++i ) + my_counters[i] = 0; + } + + size_t operator[]( int i ) { + size_t v = my_counters[i]; + return v; + } + + tbb::task * try_put_task( const T &v ) __TBB_override { + ++my_counters[(int)v]; + return const_cast(tbb::flow::internal::SUCCESSFULLY_ENQUEUED); + } + + tbb::flow::graph& graph_reference() __TBB_override { + return my_graph; + } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + typedef typename tbb::flow::receiver::built_predecessors_type built_predecessors_type; + built_predecessors_type mbp; + built_predecessors_type &built_predecessors() __TBB_override { return mbp; } + typedef typename tbb::flow::receiver::predecessor_list_type predecessor_list_type; + typedef typename tbb::flow::receiver::predecessor_type predecessor_type; + void internal_add_built_predecessor(predecessor_type &) __TBB_override {} + void internal_delete_built_predecessor(predecessor_type &) __TBB_override {} + void copy_predecessors(predecessor_list_type &) __TBB_override {} + size_t predecessor_count() __TBB_override { return 0; } +#endif + void reset_receiver(tbb::flow::reset_flags /*f*/) __TBB_override { } + +}; + +template< typename T > +void test_serial_broadcasts() { + + tbb::flow::graph g; + tbb::flow::broadcast_node b(g); + + for ( int num_receivers = 1; num_receivers < R; ++num_receivers ) { + std::vector< counting_array_receiver > receivers(num_receivers, counting_array_receiver(g)); +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + ASSERT(b.successor_count() == 0, NULL); + ASSERT(b.predecessor_count() == 0, NULL); + typename tbb::flow::broadcast_node::successor_list_type my_succs; + b.copy_successors(my_succs); + ASSERT(my_succs.size() == 0, NULL); + typename tbb::flow::broadcast_node::predecessor_list_type my_preds; + b.copy_predecessors(my_preds); + ASSERT(my_preds.size() == 0, NULL); +#endif + + for ( int r = 0; r < num_receivers; ++r ) { + tbb::flow::make_edge( b, receivers[r] ); + } +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + ASSERT( b.successor_count() == (size_t)num_receivers, NULL); +#endif + + for (int n = 0; n < N; ++n ) { + ASSERT( b.try_put( (T)n ), NULL ); + } + + for ( int r = 0; r < num_receivers; ++r ) { + for (int n = 0; n < N; ++n ) { + ASSERT( receivers[r][n] == 1, NULL ); + } + tbb::flow::remove_edge( b, receivers[r] ); + } + ASSERT( b.try_put( (T)0 ), NULL ); + for ( int r = 0; r < num_receivers; ++r ) + ASSERT( receivers[0][0] == 1, NULL ); + } + +} + +template< typename T > +class native_body : private NoAssign { + + tbb::flow::broadcast_node &my_b; + +public: + + native_body( tbb::flow::broadcast_node &b ) : my_b(b) {} + + void operator()(int) const { + for (int n = 0; n < N; ++n ) { + ASSERT( my_b.try_put( (T)n ), NULL ); + } + } + +}; + +template< typename T > +void run_parallel_broadcasts(tbb::flow::graph& g, int p, tbb::flow::broadcast_node& b) { + for ( int num_receivers = 1; num_receivers < R; ++num_receivers ) { + std::vector< counting_array_receiver > receivers(num_receivers, counting_array_receiver(g)); + + for ( int r = 0; r < num_receivers; ++r ) { + tbb::flow::make_edge( b, receivers[r] ); + } + + NativeParallelFor( p, native_body( b ) ); + + for ( int r = 0; r < num_receivers; ++r ) { + for (int n = 0; n < N; ++n ) { + ASSERT( (int)receivers[r][n] == p, NULL ); + } + tbb::flow::remove_edge( b, receivers[r] ); + } + ASSERT( b.try_put( (T)0 ), NULL ); + for ( int r = 0; r < num_receivers; ++r ) + ASSERT( (int)receivers[r][0] == p, NULL ); + } +} + +template< typename T > +void test_parallel_broadcasts(int p) { + + tbb::flow::graph g; + tbb::flow::broadcast_node b(g); + run_parallel_broadcasts(g, p, b); + + // test copy constructor + tbb::flow::broadcast_node b_copy(b); + run_parallel_broadcasts(g, p, b_copy); +} + +// broadcast_node does not allow successors to try_get from it (it does not allow +// the flow edge to switch) so we only need test the forward direction. +template +void test_resets() { + tbb::flow::graph g; + tbb::flow::broadcast_node b0(g); + tbb::flow::broadcast_node b1(g); + tbb::flow::queue_node q0(g); + tbb::flow::make_edge(b0,b1); + tbb::flow::make_edge(b1,q0); + T j; + + // test standard reset + for(int testNo = 0; testNo < 2; ++testNo) { + for(T i= 0; i <= 3; i += 1) { + b0.try_put(i); + } + g.wait_for_all(); + for(T i= 0; i <= 3; i += 1) { + ASSERT(q0.try_get(j) && j == i, "Bad value in queue"); + } + ASSERT(!q0.try_get(j), "extra value in queue"); + + // reset the graph. It should work as before. + if (testNo == 0) g.reset(); + } + + g.reset(tbb::flow::rf_clear_edges); + for(T i= 0; i <= 3; i += 1) { + b0.try_put(i); + } + g.wait_for_all(); + ASSERT(!q0.try_get(j), "edge between nodes not removed"); + for(T i= 0; i <= 3; i += 1) { + b1.try_put(i); + } + g.wait_for_all(); + ASSERT(!q0.try_get(j), "edge between nodes not removed"); +} + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION +void test_extract() { + int dont_care; + tbb::flow::graph g; + tbb::flow::broadcast_node b0(g); + tbb::flow::broadcast_node b1(g); + tbb::flow::broadcast_node b2(g); + tbb::flow::broadcast_node b3(g); + tbb::flow::broadcast_node b4(g); + tbb::flow::broadcast_node b5(g); + tbb::flow::queue_node q0(g); + tbb::flow::make_edge(b0,b1); + tbb::flow::make_edge(b0,b2); + tbb::flow::make_edge(b1,b3); + tbb::flow::make_edge(b1,b4); + tbb::flow::make_edge(b2,b4); + tbb::flow::make_edge(b2,b5); + tbb::flow::make_edge(b3,q0); + tbb::flow::make_edge(b4,q0); + tbb::flow::make_edge(b5,q0); + + /* b3 */ + /* / \ */ + /* b1 \ */ + /* / \ \ */ + /* b0 b4---q0 */ + /* \ / / */ + /* b2 / */ + /* \ / */ + /* b5 */ + + g.wait_for_all(); + b0.try_put(1); + g.wait_for_all(); + for( int i = 0; i < 4; ++i ) { + int j; + ASSERT(q0.try_get(j) && j == 1, "missing or incorrect message"); + } + ASSERT(!q0.try_get(dont_care), "extra message in queue"); + ASSERT(b0.predecessor_count() == 0 && b0.successor_count() == 2, "improper count for b0"); + ASSERT(b1.predecessor_count() == 1 && b1.successor_count() == 2, "improper count for b1"); + ASSERT(b2.predecessor_count() == 1 && b2.successor_count() == 2, "improper count for b2"); + ASSERT(b3.predecessor_count() == 1 && b3.successor_count() == 1, "improper count for b3"); + ASSERT(b4.predecessor_count() == 2 && b4.successor_count() == 1, "improper count before extract of b4"); + ASSERT(b5.predecessor_count() == 1 && b5.successor_count() == 1, "improper count for b5"); + b4.extract(); // remove from tree of nodes. + ASSERT(b0.predecessor_count() == 0 && b0.successor_count() == 2, "improper count for b0 after"); + ASSERT(b1.predecessor_count() == 1 && b1.successor_count() == 1, "improper succ count for b1 after"); + ASSERT(b2.predecessor_count() == 1 && b2.successor_count() == 1, "improper succ count for b2 after"); + ASSERT(b3.predecessor_count() == 1 && b3.successor_count() == 1, "improper succ count for b3 after"); + ASSERT(b4.predecessor_count() == 0 && b4.successor_count() == 0, "improper succ count after extract"); + ASSERT(b5.predecessor_count() == 1 && b5.successor_count() == 1, "improper succ count for b5 after"); + + /* b3 */ + /* / \ */ + /* b1 \ */ + /* / \ */ + /* b0 q0 */ + /* \ / */ + /* b2 / */ + /* \ / */ + /* b5 */ + + b0.try_put(1); + g.wait_for_all(); + for( int i = 0; i < 2; ++i ) { + int j; + ASSERT(q0.try_get(j) && j == 1, "missing or incorrect message"); + } + ASSERT(!q0.try_get(dont_care), "extra message in queue"); + tbb::flow::make_edge(b0,b4); + tbb::flow::make_edge(b4,q0); + g.wait_for_all(); + ASSERT(b0.predecessor_count() == 0 && b0.successor_count() == 3, "improper count for b0 after"); + ASSERT(b1.predecessor_count() == 1 && b1.successor_count() == 1, "improper succ count for b1 after"); + ASSERT(b2.predecessor_count() == 1 && b2.successor_count() == 1, "improper succ count for b2 after"); + ASSERT(b3.predecessor_count() == 1 && b3.successor_count() == 1, "improper succ count for b3 after"); + ASSERT(b4.predecessor_count() == 1 && b4.successor_count() == 1, "improper succ count after extract"); + ASSERT(b5.predecessor_count() == 1 && b5.successor_count() == 1, "improper succ count for b5 after"); + + /* b3 */ + /* / \ */ + /* b1 \ */ + /* / \ */ + /* b0---b4---q0 */ + /* \ / */ + /* b2 / */ + /* \ / */ + /* b5 */ + + b0.try_put(1); + g.wait_for_all(); + for( int i = 0; i < 3; ++i ) { + int j; + ASSERT(q0.try_get(j) && j == 1, "missing or incorrect message"); + } + ASSERT(!q0.try_get(dont_care), "extra message in queue"); +} +#endif // TBB_DEPRECATED_FLOW_NODE_EXTRACTION + +int TestMain() { + if( MinThread<1 ) { + REPORT("number of threads must be positive\n"); + exit(1); + } + + test_serial_broadcasts(); + test_serial_broadcasts(); + test_serial_broadcasts(); + + for( int p=MinThread; p<=MaxThread; ++p ) { + test_parallel_broadcasts(p); + test_parallel_broadcasts(p); + test_parallel_broadcasts(p); + } + + test_resets(); + test_resets(); +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + test_extract(); +#endif + + return Harness::Done; +} diff --git a/src/tbb/src/test/test_buffer_node.cpp b/src/tbb/src/test/test_buffer_node.cpp new file mode 100644 index 00000000..c2a8f09a --- /dev/null +++ b/src/tbb/src/test/test_buffer_node.cpp @@ -0,0 +1,442 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "harness.h" + +#if __TBB_CPF_BUILD +#define TBB_DEPRECATED_FLOW_NODE_EXTRACTION 1 +#endif + +#include "harness_graph.h" + +#include "tbb/task_scheduler_init.h" +#include "tbb/tick_count.h" + +#define N 1000 +#define C 10 + +template< typename T > +void spin_try_get( tbb::flow::buffer_node &b, T &value ) { + while ( b.try_get(value) != true ) {} +} + +template< typename T > +void check_item( T* count_value, T &value ) { + count_value[value / N] += value % N; +} + +template< typename T > +struct parallel_puts : NoAssign { + + tbb::flow::buffer_node &my_b; + + parallel_puts( tbb::flow::buffer_node &b ) : my_b(b) {} + + void operator()(int i) const { + for (int j = 0; j < N; ++j) { + bool msg = my_b.try_put( T(N*i + j) ); + ASSERT( msg == true, NULL ); + } + } +}; + +template< typename T > +struct touches { + + bool **my_touches; + int my_num_threads; + + touches( int num_threads ) : my_num_threads(num_threads) { + my_touches = new bool* [my_num_threads]; + for ( int p = 0; p < my_num_threads; ++p) { + my_touches[p] = new bool[N]; + for ( int n = 0; n < N; ++n) + my_touches[p][n] = false; + } + } + + ~touches() { + for ( int p = 0; p < my_num_threads; ++p) { + delete [] my_touches[p]; + } + delete [] my_touches; + } + + bool check( T v ) { + ASSERT ( my_touches[v/N][v%N] == false, NULL); + my_touches[v/N][v%N] = true; + return true; + } + + bool validate_touches() { + for ( int p = 0; p < my_num_threads; ++p) { + for ( int n = 0; n < N; ++n) { + ASSERT ( my_touches[p][n] == true, NULL); + } + } + return true; + } +}; + +template< typename T > +struct parallel_gets : NoAssign { + + tbb::flow::buffer_node &my_b; + touches &my_touches; + + parallel_gets( tbb::flow::buffer_node &b, touches &t) : my_b(b), my_touches(t) {} + + void operator()(int) const { + for (int j = 0; j < N; ++j) { + T v; + spin_try_get( my_b, v ); + my_touches.check( v ); + } + } + +}; + +template< typename T > +struct parallel_put_get : NoAssign { + + tbb::flow::buffer_node &my_b; + touches &my_touches; + + parallel_put_get( tbb::flow::buffer_node &b, touches &t ) : my_b(b), my_touches(t) {} + + void operator()(int tid) const { + + for ( int i = 0; i < N; i+=C ) { + int j_end = ( N < i + C ) ? N : i + C; + // dump about C values into the buffer + for ( int j = i; j < j_end; ++j ) { + ASSERT( my_b.try_put( T (N*tid + j ) ) == true, NULL ); + } + // receiver about C values from the buffer + for ( int j = i; j < j_end; ++j ) { + T v; + spin_try_get( my_b, v ); + my_touches.check( v ); + } + } + } + +}; + +// +// Tests +// +// Item can be reserved, released, consumed ( single serial receiver ) +// +template< typename T > +int test_reservation() { + tbb::flow::graph g; + T bogus_value(-1); + + // Simple tests + tbb::flow::buffer_node b(g); + + b.try_put(T(1)); + b.try_put(T(2)); + b.try_put(T(3)); + + T v, vsum; + ASSERT( b.try_reserve(v) == true, NULL ); + ASSERT( b.try_release() == true, NULL ); + v = bogus_value; + g.wait_for_all(); + ASSERT( b.try_reserve(v) == true, NULL ); + ASSERT( b.try_consume() == true, NULL ); + vsum += v; + v = bogus_value; + g.wait_for_all(); + + ASSERT( b.try_get(v) == true, NULL ); + vsum += v; + v = bogus_value; + g.wait_for_all(); + + ASSERT( b.try_reserve(v) == true, NULL ); + ASSERT( b.try_release() == true, NULL ); + v = bogus_value; + g.wait_for_all(); + ASSERT( b.try_reserve(v) == true, NULL ); + ASSERT( b.try_consume() == true, NULL ); + vsum += v; + ASSERT( vsum == T(6), NULL); + v = bogus_value; + g.wait_for_all(); + + return 0; +} + +// +// Tests +// +// multiple parallel senders, items in arbitrary order +// multiple parallel senders, multiple parallel receivers, items in arbitrary order and all items received +// * overlapped puts / gets +// * all puts finished before any getS +// +template< typename T > +int test_parallel(int num_threads) { + tbb::flow::graph g; + tbb::flow::buffer_node b(g); + tbb::flow::buffer_node b2(g); + tbb::flow::buffer_node b3(g); + T bogus_value(-1); + T j = bogus_value; + + NativeParallelFor( num_threads, parallel_puts(b) ); + + T *next_value = new T[num_threads]; + for (int tid = 0; tid < num_threads; ++tid) next_value[tid] = T(0); + + for (int i = 0; i < num_threads * N; ++i ) { + spin_try_get( b, j ); + check_item( next_value, j ); + j = bogus_value; + } + for (int tid = 0; tid < num_threads; ++tid) { + ASSERT( next_value[tid] == T((N*(N-1))/2), NULL ); + } + + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + NativeParallelFor( num_threads, parallel_puts(b) ); + + { + touches< T > t( num_threads ); + NativeParallelFor( num_threads, parallel_gets(b, t) ); + g.wait_for_all(); + ASSERT( t.validate_touches(), NULL ); + } + j = bogus_value; + ASSERT( b.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + g.wait_for_all(); + { + touches< T > t( num_threads ); + NativeParallelFor( num_threads, parallel_put_get(b, t) ); + g.wait_for_all(); + ASSERT( t.validate_touches(), NULL ); + } + j = bogus_value; + ASSERT( b.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + tbb::flow::make_edge( b, b2 ); + tbb::flow::make_edge( b2, b3 ); + + NativeParallelFor( num_threads, parallel_puts(b) ); + { + touches< T > t( num_threads ); + NativeParallelFor( num_threads, parallel_gets(b3, t) ); + g.wait_for_all(); + ASSERT( t.validate_touches(), NULL ); + } + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b2.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b3.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + // test copy constructor + ASSERT( b.remove_successor( b2 ), NULL ); + // fill up b: + NativeParallelFor( num_threads, parallel_puts(b) ); + // copy b: + tbb::flow::buffer_node b_copy(b); + + // b_copy should be empty + j = bogus_value; + g.wait_for_all(); + ASSERT( b_copy.try_get( j ) == false, NULL ); + + // hook them together: + ASSERT( b.register_successor(b_copy) == true, NULL ); + // try to get content from b_copy + { + touches< T > t( num_threads ); + NativeParallelFor( num_threads, parallel_gets(b_copy, t) ); + g.wait_for_all(); + ASSERT( t.validate_touches(), NULL ); + } + // now both should be empty + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b_copy.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + delete [] next_value; + return 0; +} + +// +// Tests +// +// Predecessors cannot be registered +// Empty buffer rejects item requests +// Single serial sender, items in arbitrary order +// Chained buffers ( 2 & 3 ), single sender, items at last buffer in arbitrary order +// + +template< typename T > +int test_serial() { + tbb::flow::graph g; + T bogus_value(-1); + + tbb::flow::buffer_node b(g); + tbb::flow::buffer_node b2(g); + T j = bogus_value; + + // + // Rejects attempts to add / remove predecessor + // Rejects request from empty buffer + // + ASSERT( b.register_predecessor( b2 ) == false, NULL ); + ASSERT( b.remove_predecessor( b2 ) == false, NULL ); + ASSERT( b.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + // + // Simple puts and gets + // + + for (int i = 0; i < N; ++i) { + bool msg = b.try_put( T(i) ); + ASSERT( msg == true, NULL ); + } + + T vsum = T(0); + for (int i = 0; i < N; ++i) { + j = bogus_value; + spin_try_get( b, j ); + vsum += j; + } + ASSERT( vsum == (N*(N-1))/2, NULL); + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + tbb::flow::make_edge(b, b2); +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + ASSERT( b.successor_count() == 1, NULL); + ASSERT( b.predecessor_count() == 0, NULL); + ASSERT( b2.successor_count() == 0, NULL); + ASSERT( b2.predecessor_count() == 1, NULL); + typename tbb::flow::buffer_node::successor_list_type my_succs; + b.copy_successors(my_succs); + ASSERT(my_succs.size() == 1, NULL); + typename tbb::flow::buffer_node::predecessor_list_type my_preds; + b.copy_predecessors(my_preds); + ASSERT(my_preds.size() == 0, NULL); +#endif + + vsum = T(0); + for (int i = 0; i < N; ++i) { + bool msg = b.try_put( T(i) ); + ASSERT( msg == true, NULL ); + } + + for (int i = 0; i < N; ++i) { + j = bogus_value; + spin_try_get( b2, j ); + vsum += j; + } + ASSERT( vsum == (N*(N-1))/2, NULL); + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b2.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + tbb::flow::remove_edge(b, b2); + ASSERT( b.try_put( 1 ) == true, NULL ); + g.wait_for_all(); + ASSERT( b2.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + g.wait_for_all(); + ASSERT( b.try_get( j ) == true, NULL ); + ASSERT( j == 1, NULL ); + + tbb::flow::buffer_node b3(g); + tbb::flow::make_edge( b, b2 ); + tbb::flow::make_edge( b2, b3 ); + + vsum = T(0); + for (int i = 0; i < N; ++i) { + bool msg = b.try_put( T(i) ); + ASSERT( msg == true, NULL ); + } + + for (int i = 0; i < N; ++i) { + j = bogus_value; + spin_try_get( b3, j ); + vsum += j; + } + ASSERT( vsum == (N*(N-1))/2, NULL); + j = bogus_value; + g.wait_for_all(); + ASSERT( b.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b2.try_get( j ) == false, NULL ); + g.wait_for_all(); + ASSERT( b3.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + + tbb::flow::remove_edge(b, b2); + ASSERT( b.try_put( 1 ) == true, NULL ); + g.wait_for_all(); + ASSERT( b2.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + g.wait_for_all(); + ASSERT( b3.try_get( j ) == false, NULL ); + ASSERT( j == bogus_value, NULL ); + g.wait_for_all(); + ASSERT( b.try_get( j ) == true, NULL ); + ASSERT( j == 1, NULL ); + + return 0; +} + +int TestMain() { + tbb::tick_count start = tbb::tick_count::now(), stop; + for (int p = 2; p <= 4; ++p) { + tbb::task_scheduler_init init(p); + test_serial(); + test_parallel(p); + } + stop = tbb::tick_count::now(); + REMARK("Buffer_Node Time=%6.6f\n", (stop-start).seconds()); + test_resets >(); + test_resets >(); +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + test_buffer_extract >().run_tests(); +#endif + return Harness::Done; +} diff --git a/src/tbb/src/test/test_cache_aligned_allocator.cpp b/src/tbb/src/test/test_cache_aligned_allocator.cpp new file mode 100644 index 00000000..b5e65bef --- /dev/null +++ b/src/tbb/src/test/test_cache_aligned_allocator.cpp @@ -0,0 +1,76 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Test whether cache_aligned_allocator works with some of the host's STL containers. + +#include "tbb/cache_aligned_allocator.h" +#include "tbb/tbb_allocator.h" + +#define HARNESS_NO_PARSE_COMMAND_LINE 1 +// the real body of the test is there: +#include "test_allocator.h" + +template<> +struct is_zero_filling > { + static const bool value = true; +}; + +// Test that NFS_Allocate() throws bad_alloc if cannot allocate memory. +void Test_NFS_Allocate_Throws() { +#if TBB_USE_EXCEPTIONS && !__TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN + using namespace tbb::internal; + + // First, allocate a reasonably big amount of memory, big enough + // to not cause warp around in system allocator after adding object header + // during address2 allocation. + const size_t itemsize = 1024; + const size_t nitems = 1024; + void *address1 = NULL; + try { + address1 = NFS_Allocate( nitems, itemsize, NULL ); + } catch( ... ) { + // intentionally empty + } + ASSERT( address1, "NFS_Allocate unable to obtain 1024*1024 bytes" ); + + bool exception_caught = false; + try { + // Try allocating more memory than left in the address space; should cause std::bad_alloc + (void) NFS_Allocate( 1, ~size_t(0) - itemsize*nitems + NFS_GetLineSize(), NULL); + } catch( std::bad_alloc& ) { + exception_caught = true; + } catch( ... ) { + ASSERT( __TBB_EXCEPTION_TYPE_INFO_BROKEN, "Unexpected exception type (std::bad_alloc was expected)" ); + exception_caught = true; + } + ASSERT( exception_caught, "NFS_Allocate did not throw bad_alloc" ); + + try { + NFS_Free( address1 ); + } catch( ... ) { + ASSERT( false, "NFS_Free did not accept the address obtained with NFS_Allocate" ); + } +#endif /* TBB_USE_EXCEPTIONS && !__TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN */ +} + +int TestMain () { + int result = TestMain >(); + result += TestMain >(); + result += TestMain >(); + ASSERT( !result, NULL ); + Test_NFS_Allocate_Throws(); + return Harness::Done; +} diff --git a/src/tbb/src/test/test_cache_aligned_allocator_STL.cpp b/src/tbb/src/test/test_cache_aligned_allocator_STL.cpp new file mode 100644 index 00000000..eef664ba --- /dev/null +++ b/src/tbb/src/test/test_cache_aligned_allocator_STL.cpp @@ -0,0 +1,42 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Test whether cache_aligned_allocator works with some of the host's STL containers. + +#include "tbb/cache_aligned_allocator.h" +#include "tbb/tbb_allocator.h" + +#define HARNESS_NO_PARSE_COMMAND_LINE 1 +#include "test_allocator_STL.h" + +int TestMain () { + TestAllocatorWithSTL >(); + TestAllocatorWithSTL >(); + TestAllocatorWithSTL >(); + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + tbb::cache_aligned_resource aligned_resource; + tbb::cache_aligned_resource equal_aligned_resource(std::pmr::get_default_resource()); + ASSERT(aligned_resource.is_equal(equal_aligned_resource), + "Underlying upstream resources should be equal."); + ASSERT(!aligned_resource.is_equal(*std::pmr::null_memory_resource()), + "Cache aligned resource upstream shouldn't be equal to the standard resource."); + TestAllocatorWithSTL(std::pmr::polymorphic_allocator(&aligned_resource)); +#endif + + return Harness::Done; +} + diff --git a/src/tbb/src/test/test_cilk_common.h b/src/tbb/src/test/test_cilk_common.h new file mode 100644 index 00000000..c5b1f079 --- /dev/null +++ b/src/tbb/src/test/test_cilk_common.h @@ -0,0 +1,79 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// This file is a common part of test_cilk_interop and test_cilk_dynamic_load tests + +int TBB_Fib( int n ); + +class FibCilkSubtask: public tbb::task { + int n; + int& result; + task* execute() __TBB_override { + if( n<2 ) { + result = n; + } else { + int x, y; + x = cilk_spawn TBB_Fib(n-2); + y = cilk_spawn TBB_Fib(n-1); + cilk_sync; + result = x+y; + } + return NULL; + } +public: + FibCilkSubtask( int& result_, int n_ ) : result(result_), n(n_) {} +}; + +class FibTask: public tbb::task { + int n; + int& result; + task* execute() __TBB_override { + if( !g_sandwich && n<2 ) { + result = n; + } else { + int x,y; + tbb::task_scheduler_init init(P_nested); + task* self0 = &task::self(); + set_ref_count( 3 ); + if ( g_sandwich ) { + spawn (*new( allocate_child() ) FibCilkSubtask(x,n-1)); + spawn (*new( allocate_child() ) FibCilkSubtask(y,n-2)); + } + else { + spawn (*new( allocate_child() ) FibTask(x,n-1)); + spawn (*new( allocate_child() ) FibTask(y,n-2)); + } + wait_for_all(); + task* self1 = &task::self(); + ASSERT( self0 == self1, "failed to preserve TBB TLS" ); + result = x+y; + } + return NULL; + } +public: + FibTask( int& result_, int n_ ) : result(result_), n(n_) {} +}; + +int TBB_Fib( int n ) { + if( n<2 ) { + return n; + } else { + int result; + tbb::task_scheduler_init init(P_nested); + tbb::task::spawn_root_and_wait(*new( tbb::task::allocate_root()) FibTask(result,n) ); + return result; + } +} diff --git a/src/tbb/src/test/test_cilk_dynamic_load.cpp b/src/tbb/src/test/test_cilk_dynamic_load.cpp new file mode 100644 index 00000000..10543633 --- /dev/null +++ b/src/tbb/src/test/test_cilk_dynamic_load.cpp @@ -0,0 +1,152 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "tbb/tbb_config.h" + +// Skip the test if no interoperability with cilkrts +#define __TBB_CILK_INTEROP (__TBB_SURVIVE_THREAD_SWITCH && __INTEL_COMPILER>=1200) +// Skip the test when cilkrts did not have dlopen()/dlclose() start up feature +#define CILK_SYMBOLS_VISIBLE (_WIN32||_WIN64) +// The compiler does not add "-lcilkrts" linker option on some linux systems +#define CILK_LINKAGE_BROKEN (__linux__ && __GNUC__<4 && __INTEL_COMPILER_BUILD_DATE <= 20110427) +// Currently, the interop doesn't support the situation: +//1) Intel(R) Threading Building Blocks (Intel(R) TBB) is outermost; +//2) Intel(R) Cilk(TM) Plus, and it should be dynamically loaded with dlopen/LoadLibrary (possibly via a 3rd party module); +//3) Intel(R) TBB again; +//4) Intel(R) Cilk(TM) Plus again. +#define HEAVY_NESTED_INTEROP_SUPPORT ( __INTEL_COMPILER_BUILD_DATE < 20110427 ) + +#if __TBB_CILK_INTEROP && CILK_SYMBOLS_VISIBLE && !CILK_LINKAGE_BROKEN && HEAVY_NESTED_INTEROP_SUPPORT + +#include "tbb/task_scheduler_init.h" +#include "tbb/task.h" + +static const int N = 25; +static const int P_outer = 4; +static const int P_nested = 2; + +#ifdef _USRDLL + +#include +#define HARNESS_CUSTOM_MAIN 1 +#include "harness.h" +#undef HARNESS_CUSTOM_MAIN + +#if _WIN32 || _WIN64 +#define CILK_TEST_EXPORT extern "C" __declspec(dllexport) +#else +#define CILK_TEST_EXPORT extern "C" +#endif /* _WIN32 || _WIN64 */ + +bool g_sandwich = true; // have to be declare before #include "test_cilk_common.h" +#include "test_cilk_common.h" + +CILK_TEST_EXPORT int CilkFib( int n ) +{ + return TBB_Fib(n); +} + +CILK_TEST_EXPORT void CilkShutdown() +{ + __cilkrts_end_cilk(); +} + +#else /* _USRDLL undefined */ + +#include "harness.h" +#include "harness_dynamic_libs.h" + +int SerialFib( int n ) { + int a=0, b=1; + for( int i=0; i=1200) +// The compiler does not add "-lcilkrts" linker option on some linux systems +#define CILK_LINKAGE_BROKEN (__linux__ && __GNUC__<4 && __INTEL_COMPILER_BUILD_DATE <= 20110427) +// In U4, cilkrts incorrectly sends the interop notifications to TBB +#define CILK_NOTIFICATIONS_BROKEN ( __INTEL_COMPILER_BUILD_DATE == 20110427 ) + +#if __TBB_CILK_INTEROP && !CILK_LINKAGE_BROKEN && !CILK_NOTIFICATIONS_BROKEN + +static const int N = 14; +static const int P_outer = 4; +static const int P_nested = 2; + +#include +#include +#define private public +#include "tbb/task.h" +#undef private +#include "tbb/task_scheduler_init.h" +#include +#include + +enum tbb_sched_injection_mode_t { + tbbsched_none = 0, + tbbsched_explicit_only = 1, + tbbsched_auto_only = 2, + tbbsched_mixed = 3 +}; + +tbb_sched_injection_mode_t g_sim = tbbsched_none; + +bool g_sandwich = false; // have to be declare before #include "test_cilk_common.h" +#include "test_cilk_common.h" + +// A time delay routine +void Delay( int n ) { + static volatile int Global; + for( int k=0; k<10000; ++k ) + for( int i=0; i +#include +#include + +#include "harness_assert.h" +#include "harness.h" +#include "test_container_move_support.h" + +#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif + +static tbb::atomic construction_counter; +static tbb::atomic destruction_counter; + +const int REPETITIONS = 10; +const int N = 100000; +const double EXPECTED_SUM = (REPETITIONS + 1) * N; + +// +// A minimal class +// Define: default and copy constructor, and allow implicit operator& +// also operator= +// + +class minimal { +private: + int my_value; +public: + minimal(int val=0) : my_value(val) { ++construction_counter; } + minimal( const minimal &m ) : my_value(m.my_value) { ++construction_counter; } + minimal& operator=(const minimal& other) { my_value = other.my_value; return *this; } + minimal& operator+=(const minimal& other) { my_value += other.my_value; return *this; } + operator int() const { return my_value; } + ~minimal() { ++destruction_counter; } + void set_value( const int i ) { my_value = i; } + int value( ) const { return my_value; } +}; + +//// functors for initialization and combine + +template +struct FunctorAddFinit { + T operator()() { return 0; } +}; + +template +struct FunctorAddFinit7 { + T operator()() { return 7; } +}; + +template +struct FunctorAddCombine { + T operator()(T left, T right ) const { + return left + right; + } +}; + +template +struct FunctorAddCombineRef { + T operator()(const T& left, const T& right ) const { + return left + right; + } +}; + +template +T my_combine( T left, T right) { return left + right; } + +template +T my_combine_ref( const T &left, const T &right) { return left + right; } + +template +class CombineEachHelper { +public: + CombineEachHelper(T& _result) : my_result(_result) {} + void operator()(const T& new_bit) { my_result += new_bit; } + CombineEachHelper& operator=(const CombineEachHelper& other) { + my_result = other; + return *this; + } +private: + T& my_result; +}; + +template +class CombineEachHelperCnt { +public: + CombineEachHelperCnt(T& _result, int& _nbuckets) : my_result(_result), nBuckets(_nbuckets) {} + void operator()(const T& new_bit) { my_result += new_bit; ++nBuckets; } + CombineEachHelperCnt& operator=(const CombineEachHelperCnt& other) { + my_result = other.my_result; + nBuckets = other.nBuckets; + return *this; + } +private: + T& my_result; + int& nBuckets; +}; + +template +class CombineEachVectorHelper { +public: + typedef std::vector > ContainerType; + CombineEachVectorHelper(T& _result) : my_result(_result) { } + void operator()(const ContainerType& new_bit) { + for(typename ContainerType::const_iterator ci = new_bit.begin(); ci != new_bit.end(); ++ci) { + my_result += *ci; + } + } + CombineEachVectorHelper& operator=(const CombineEachVectorHelper& other) { + my_result=other.my_result; + return *this; + } + +private: + T& my_result; +}; + +//// end functors + +// parallel body with a test for first access +template +class ParallelScalarBody: NoAssign { + + tbb::combinable &sums; + +public: + + ParallelScalarBody ( tbb::combinable &_sums ) : sums(_sums) { } + + void operator()( const tbb::blocked_range &r ) const { + for (int i = r.begin(); i != r.end(); ++i) { + bool was_there; + T& my_local = sums.local(was_there); + if(!was_there) my_local = 0; + my_local += 1 ; + } + } + +}; + +// parallel body with no test for first access +template +class ParallelScalarBodyNoInit: NoAssign { + + tbb::combinable &sums; + +public: + + ParallelScalarBodyNoInit ( tbb::combinable &_sums ) : sums(_sums) { } + + void operator()( const tbb::blocked_range &r ) const { + for (int i = r.begin(); i != r.end(); ++i) { + sums.local() += 1 ; + } + } + +}; + +template< typename T > +void RunParallelScalarTests(const char *test_name) { + + tbb::task_scheduler_init init(tbb::task_scheduler_init::deferred); + for (int p = MinThread; p <= MaxThread; ++p) { + + if (p == 0) continue; + REMARK(" Testing parallel %s on %d thread(s)...\n", test_name, p); + init.initialize(p); + + tbb::tick_count t0; + T combine_sum(0); + T combine_ref_sum(0); + T combine_finit_sum(0); + T combine_each_sum(0); + T copy_construct_sum(0); + T copy_assign_sum(0); +#if __TBB_ETS_USE_CPP11 + T move_construct_sum(0); + T move_assign_sum(0); +#endif + for (int t = -1; t < REPETITIONS; ++t) { + if (Verbose && t == 0) t0 = tbb::tick_count::now(); + + // test uninitialized parallel combinable + tbb::combinable sums; + tbb::parallel_for( tbb::blocked_range( 0, N, 10000 ), ParallelScalarBody( sums ) ); + combine_sum += sums.combine(my_combine); + combine_ref_sum += sums.combine(my_combine_ref); + + // test parallel combinable preinitialized with a functor that returns 0 + FunctorAddFinit my_finit_decl; + tbb::combinable finit_combinable(my_finit_decl); + tbb::parallel_for( tbb::blocked_range( 0, N, 10000 ), ParallelScalarBodyNoInit( finit_combinable ) ); + combine_finit_sum += finit_combinable.combine(my_combine); + + // test another way of combining the elements using CombineEachHelper functor + CombineEachHelper my_helper(combine_each_sum); + sums.combine_each(my_helper); + + // test copy constructor for parallel combinable + tbb::combinable copy_constructed(sums); + copy_construct_sum += copy_constructed.combine(my_combine); + + // test copy assignment for uninitialized parallel combinable + tbb::combinable assigned; + assigned = sums; + copy_assign_sum += assigned.combine(my_combine); + +#if __TBB_ETS_USE_CPP11 + // test move constructor for parallel combinable + tbb::combinable moved1(std::move(sums)); + move_construct_sum += moved1.combine(my_combine); + + // test move assignment for uninitialized parallel combinable + tbb::combinable moved2; + moved2=std::move(finit_combinable); + move_assign_sum += moved2.combine(my_combine); +#endif + } + // Here and below comparison for equality of float numbers succeeds + // as the rounding error doesn't accumulate and doesn't affect the comparison + ASSERT( EXPECTED_SUM == combine_sum, NULL); + ASSERT( EXPECTED_SUM == combine_ref_sum, NULL); + ASSERT( EXPECTED_SUM == combine_finit_sum, NULL); + ASSERT( EXPECTED_SUM == combine_each_sum, NULL); + ASSERT( EXPECTED_SUM == copy_construct_sum, NULL); + ASSERT( EXPECTED_SUM == copy_assign_sum, NULL); +#if __TBB_ETS_USE_CPP11 + ASSERT( EXPECTED_SUM == move_construct_sum, NULL); + ASSERT( EXPECTED_SUM == move_assign_sum, NULL); +#endif + REMARK(" done parallel %s, %d, %g, %g\n", test_name, p, static_cast(combine_sum), + ( tbb::tick_count::now() - t0).seconds()); + init.terminate(); + } +} + +template +class ParallelVectorForBody: NoAssign { + + tbb::combinable< std::vector > > &locals; + +public: + + ParallelVectorForBody ( tbb::combinable< std::vector > > &_locals ) : locals(_locals) { } + + void operator()( const tbb::blocked_range &r ) const { + T one = 1; + + for (int i = r.begin(); i < r.end(); ++i) { + locals.local().push_back( one ); + } + } + +}; + +template< typename T > +void RunParallelVectorTests(const char *test_name) { + + tbb::task_scheduler_init init(tbb::task_scheduler_init::deferred); + + typedef std::vector > ContainerType; + + for (int p = MinThread; p <= MaxThread; ++p) { + + if (p == 0) continue; + REMARK(" Testing parallel %s on %d thread(s)... \n", test_name, p); + init.initialize(p); + + tbb::tick_count t0; + T defaultConstructed_sum(0); + T copyConstructed_sum(0); + T copyAssigned_sum(0); +#if __TBB_ETS_USE_CPP11 + T moveConstructed_sum(0); + T moveAssigned_sum(0); +#endif + for (int t = -1; t < REPETITIONS; ++t) { + if (Verbose && t == 0) t0 = tbb::tick_count::now(); + + typedef typename tbb::combinable< ContainerType > CombinableType; + + // test uninitialized parallel combinable + CombinableType vs; + tbb::parallel_for( tbb::blocked_range (0, N, 10000), ParallelVectorForBody( vs ) ); + CombineEachVectorHelper MyCombineEach(defaultConstructed_sum); + vs.combine_each(MyCombineEach); // combine_each sums all elements of each vector into the result + + // test copy constructor for parallel combinable with vectors + CombinableType vs2(vs); + CombineEachVectorHelper MyCombineEach2(copyConstructed_sum); + vs2.combine_each(MyCombineEach2); + + // test copy assignment for uninitialized parallel combinable with vectors + CombinableType vs3; + vs3 = vs; + CombineEachVectorHelper MyCombineEach3(copyAssigned_sum); + vs3.combine_each(MyCombineEach3); + +#if __TBB_ETS_USE_CPP11 + // test move constructor for parallel combinable with vectors + CombinableType vs4(std::move(vs2)); + CombineEachVectorHelper MyCombineEach4(moveConstructed_sum); + vs4.combine_each(MyCombineEach4); + + // test move assignment for uninitialized parallel combinable with vectors + vs4=std::move(vs3); + CombineEachVectorHelper MyCombineEach5(moveAssigned_sum); + vs4.combine_each(MyCombineEach5); +#endif + } + + double ResultValue = defaultConstructed_sum; + ASSERT( EXPECTED_SUM == ResultValue, NULL); + ResultValue = copyConstructed_sum; + ASSERT( EXPECTED_SUM == ResultValue, NULL); + ResultValue = copyAssigned_sum; + ASSERT( EXPECTED_SUM == ResultValue, NULL); +#if __TBB_ETS_USE_CPP11 + ResultValue = moveConstructed_sum; + ASSERT( EXPECTED_SUM == ResultValue, NULL); + ResultValue = moveAssigned_sum; + ASSERT( EXPECTED_SUM == ResultValue, NULL); +#endif + REMARK(" done parallel %s, %d, %g, %g\n", test_name, p, ResultValue, ( tbb::tick_count::now() - t0).seconds()); + init.terminate(); + } +} + +void +RunParallelTests() { + REMARK("Running RunParallelTests\n"); + RunParallelScalarTests("int"); + RunParallelScalarTests("double"); + RunParallelScalarTests("minimal"); + RunParallelVectorTests("std::vector >"); + RunParallelVectorTests("std::vector >"); +} + +template +void +RunAssignmentAndCopyConstructorTest(const char *test_name) { + REMARK(" Testing assignment and copy construction for combinable<%s>...\n", test_name); + + // test creation with finit function (combine returns finit return value if no threads have created locals) + FunctorAddFinit7 my_finit7_decl; + tbb::combinable create1(my_finit7_decl); + ASSERT(7 == create1.combine(my_combine), "Unexpected combine result for combinable object preinitialized with functor"); + + // test copy construction with function initializer + tbb::combinable copy1(create1); + ASSERT(7 == copy1.combine(my_combine), "Unexpected combine result for copy-constructed combinable object"); + + // test copy assignment with function initializer + FunctorAddFinit my_finit_decl; + tbb::combinable assign1(my_finit_decl); + assign1 = create1; + ASSERT(7 == assign1.combine(my_combine), "Unexpected combine result for copy-assigned combinable object"); + +#if __TBB_ETS_USE_CPP11 + // test move construction with function initializer + tbb::combinable move1(std::move(create1)); + ASSERT(7 == move1.combine(my_combine), "Unexpected combine result for move-constructed combinable object"); + + // test move assignment with function initializer + tbb::combinable move2; + move2=std::move(copy1); + ASSERT(7 == move2.combine(my_combine), "Unexpected combine result for move-assigned combinable object"); +#endif + + REMARK(" done\n"); + +} + +void +RunAssignmentAndCopyConstructorTests() { + REMARK("Running assignment and copy constructor tests:\n"); + RunAssignmentAndCopyConstructorTest("int"); + RunAssignmentAndCopyConstructorTest("double"); + RunAssignmentAndCopyConstructorTest("minimal"); +} + +void +RunMoveSemanticsForStateTrackableObjectTest() { + REMARK("Testing move assignment and move construction for combinable...\n"); + + tbb::combinable< Harness::StateTrackable > create1; + ASSERT(create1.local().state == Harness::StateTrackable::DefaultInitialized, + "Unexpected value in default combinable object"); + + // Copy constructing of the new combinable causes copying of stored values + tbb::combinable< Harness::StateTrackable > copy1(create1); + ASSERT(copy1.local().state == Harness::StateTrackable::CopyInitialized, + "Unexpected value in copy-constructed combinable object"); + + // Copy assignment also causes copying of stored values + tbb::combinable< Harness::StateTrackable > copy2; + ASSERT(copy2.local().state == Harness::StateTrackable::DefaultInitialized, + "Unexpected value in default combinable object"); + copy2=create1; + ASSERT(copy2.local().state == Harness::StateTrackable::CopyInitialized, + "Unexpected value in copy-assigned combinable object"); + +#if __TBB_ETS_USE_CPP11 + // Store some marked values in the initial combinable object + create1.local().state = Harness::StateTrackableBase::Unspecified; + + // Move constructing of the new combinable must not cause copying of stored values + tbb::combinable< Harness::StateTrackable > move1(std::move(create1)); + ASSERT(move1.local().state == Harness::StateTrackableBase::Unspecified, "Unexpected value in move-constructed combinable object"); + + // Move assignment must not cause copying of stored values + copy1=std::move(move1); + ASSERT(copy1.local().state == Harness::StateTrackableBase::Unspecified, "Unexpected value in move-assigned combinable object"); + + // Make the stored values valid again in order to delete StateTrackable object correctly + copy1.local().state = Harness::StateTrackable::MoveAssigned; +#endif + + REMARK("done\n"); +} + +#include "harness_barrier.h" + +Harness::SpinBarrier sBarrier; + +struct Body : NoAssign { + tbb::combinable* locals; + const int nthread; + const int nIters; + Body( int nthread_, int niters_ ) : nthread(nthread_), nIters(niters_) { sBarrier.initialize(nthread_); } + + void operator()(int thread_id ) const { + bool existed; + sBarrier.wait(); + for(int i = 0; i < nIters; ++i ) { + existed = thread_id & 1; + int oldval = locals->local(existed); + ASSERT(existed == (i > 0), "Error on first reference"); + ASSERT(!existed || (oldval == thread_id), "Error on fetched value"); + existed = thread_id & 1; + locals->local(existed) = thread_id; + ASSERT(existed, "Error on assignment"); + } + } +}; + +void +TestLocalAllocations( int nthread ) { + ASSERT(nthread > 0, "nthread must be positive"); +#define NITERATIONS 1000 + Body myBody(nthread, NITERATIONS); + tbb::combinable myCombinable; + myBody.locals = &myCombinable; + + NativeParallelFor( nthread, myBody ); + + int mySum = 0; + int mySlots = 0; + CombineEachHelperCnt myCountCombine(mySum, mySlots); + myCombinable.combine_each(myCountCombine); + + ASSERT(nthread == mySlots, "Incorrect number of slots"); + ASSERT(mySum == (nthread - 1) * nthread / 2, "Incorrect values in result"); +} + +void +RunLocalAllocationsTests() { + REMARK("Testing local() allocations\n"); + for(int i = 1 <= MinThread ? MinThread : 1; i <= MaxThread; ++i) { + REMARK(" Testing local() allocation with nthreads=%d...\n", i); + for(int j = 0; j < 100; ++j) { + TestLocalAllocations(i); + } + REMARK(" done\n"); + } +} + +int TestMain () { + if (MaxThread > 0) { + RunParallelTests(); + } + RunAssignmentAndCopyConstructorTests(); + RunMoveSemanticsForStateTrackableObjectTest(); + RunLocalAllocationsTests(); + return Harness::Done; +} + diff --git a/src/tbb/src/test/test_composite_node.cpp b/src/tbb/src/test/test_composite_node.cpp new file mode 100644 index 00000000..14b6b71e --- /dev/null +++ b/src/tbb/src/test/test_composite_node.cpp @@ -0,0 +1,586 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "harness.h" +#if __TBB_FLOW_GRAPH_CPP11_FEATURES + +#include "tbb/flow_graph.h" +#include "harness_graph.h" +#include +#include +#include + +struct passthru_body { + int operator()( int i ) { + return i; + } +}; + +class src_body{ + int start; + int finish; + int step; +public: + src_body(int f, int s) : start(1), finish(f), step(s) {} + bool operator()(int &a) { + a = start; + if (start <= finish) { + a = start; + start+=step; + return true; + } + else { + return false; + }; + } +}; + +struct m_fxn_body{ + void operator()(int, tbb::flow::multifunction_node >::output_ports_type ) {} +}; + +struct ct_body { +ct_body(){} + void operator()(tbb::flow::continue_msg){} +}; + +struct seq_body { +int operator()(int i){return i;} +}; + +template +struct compare { + static void compare_refs(T1 tuple1, T2 tuple2) { + ASSERT( &tbb::flow::get(tuple1) == &tbb::flow::get(tuple2), "ports not set correctly"); + compare::compare_refs(tuple1, tuple2); + } +}; + +template +struct compare<1, T1, T2> { + static void compare_refs(T1 tuple1, T2 tuple2) { + ASSERT(&tbb::flow::get<0>(tuple1) == &tbb::flow::get<0>(tuple2), "port 0 not correctly set"); + } +}; + +void add_all_nodes (){ + tbb::flow::graph g; + + typedef tbb::flow::tuple, int, int, int, int, + int, int, int, int, int, int, int, int > InputTupleType; + + typedef tbb::flow::tuple, tbb::flow::tagged_msg, + int, int, int, int, int, int, int, int, int, int, int, int > OutputTupleType; + + typedef tbb::flow::tuple< > EmptyTupleType; + + typedef tbb::flow::composite_node input_output_type; + typedef tbb::flow::composite_node input_only_type; + typedef tbb::flow::composite_node output_only_type; + + const size_t NUM_INPUTS = tbb::flow::tuple_size::value; + const size_t NUM_OUTPUTS = tbb::flow::tuple_size::value; + + //node types + tbb::flow::continue_node ct(g, ct_body()); + tbb::flow::split_node< tbb::flow::tuple > s(g); + tbb::flow::source_node src(g, src_body(20,5), false); + tbb::flow::function_node fxn(g, tbb::flow::unlimited, passthru_body()); + tbb::flow::multifunction_node > m_fxn(g, tbb::flow::unlimited, m_fxn_body()); + tbb::flow::broadcast_node bc(g); + tbb::flow::limiter_node lim(g, 2); + tbb::flow::indexer_node ind(g); + tbb::flow::join_node< tbb::flow::tuple< int, int >, tbb::flow::queueing > j(g); + tbb::flow::queue_node q(g); + tbb::flow::buffer_node bf(g); + tbb::flow::priority_queue_node pq(g); + tbb::flow::write_once_node wo(g); + tbb::flow::overwrite_node ovw(g); + tbb::flow::sequencer_node seq(g, seq_body()); + +#if !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN + auto input_tuple = std::tie(ct, s, m_fxn, fxn, bc, tbb::flow::input_port<0>(j), lim, q, tbb::flow::input_port<0>(ind), + pq, ovw, wo, bf, seq); + auto output_tuple = std::tie(ct,j, ind, fxn, src, bc, tbb::flow::output_port<0>(s), lim, tbb::flow::output_port<0>(m_fxn), + q, pq, ovw, wo, bf, seq ); +#else + // upcasting from derived to base for a tuple of references created by std::tie + // fails on gcc 4.4 (and all icc in that environment) + input_output_type::input_ports_type input_tuple(ct, s, m_fxn, fxn, bc, tbb::flow::input_port<0>(j), lim, q, + tbb::flow::input_port<0>(ind), pq, ovw, wo, bf, seq); + + input_output_type::output_ports_type output_tuple(ct,j, ind, fxn, src, bc, tbb::flow::output_port<0>(s), + lim, tbb::flow::output_port<0>(m_fxn), q, pq, ovw, wo, bf, seq); +#endif + + //composite_node with both input_ports and output_ports + input_output_type a_node(g); + a_node.set_external_ports(input_tuple, output_tuple); + + a_node.add_visible_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + a_node.add_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + + auto a_node_input_ports_ptr = a_node.input_ports(); + compare::compare_refs(a_node_input_ports_ptr, input_tuple); + ASSERT (NUM_INPUTS == tbb::flow::tuple_size::value, "not all declared input ports were bound to nodes"); + + auto a_node_output_ports_ptr = a_node.output_ports(); + compare::compare_refs(a_node_output_ports_ptr, output_tuple); + ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "not all declared output ports were bound to nodes"); + + //composite_node with only input_ports + input_only_type b_node(g); + b_node.set_external_ports(input_tuple); + + b_node.add_visible_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + b_node.add_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + + auto b_node_input_ports_ptr = b_node.input_ports(); + compare::compare_refs(b_node_input_ports_ptr, input_tuple); + ASSERT (NUM_INPUTS == tbb::flow::tuple_size::value, "not all declared input ports were bound to nodes"); + + //composite_node with only output_ports + output_only_type c_node(g); + c_node.set_external_ports(output_tuple); + + c_node.add_visible_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + + c_node.add_nodes(src, fxn, m_fxn, bc, lim, ind, s, ct, j, q, bf, pq, wo, ovw, seq); + + auto c_node_output_ports_ptr = c_node.output_ports(); + compare::compare_refs(c_node_output_ports_ptr, output_tuple); + ASSERT (NUM_OUTPUTS == tbb::flow::tuple_size::value, "not all declared input ports were bound to nodes"); +} + +struct tiny_node : public tbb::flow::composite_node< tbb::flow::tuple< int >, tbb::flow::tuple< int > > { + tbb::flow::function_node< int, int > f1; + tbb::flow::function_node< int, int > f2; + typedef tbb::flow::composite_node< tbb::flow::tuple< int >, tbb::flow::tuple< int > > base_type; + +public: + tiny_node(tbb::flow::graph &g, bool hidden = false) : base_type(g), f1(g, tbb::flow::unlimited, passthru_body() ), f2(g, tbb::flow::unlimited, passthru_body() ) { + tbb::flow::make_edge( f1, f2 ); + + tbb::flow::tuple& > input_tuple(f1); + tbb::flow::tuple& > output_tuple(f2); + base_type::set_external_ports( input_tuple, output_tuple ); + + if(hidden) + base_type::add_nodes(f1, f2); + else + base_type::add_visible_nodes(f1, f2); + + } +}; + +int test_tiny(bool hidden = false) { + tbb::flow::graph g; + tbb::flow::function_node< int, int > f0( g, tbb::flow::unlimited, passthru_body() ); + tiny_node t(g, hidden); + ASSERT(&tbb::flow::input_port<0>(t) == &t.f1, "f1 not bound to input port 0 in composite_node t"); + ASSERT(&tbb::flow::output_port<0>(t) == &t.f2, "f2 not bound to output port 0 in composite_node t"); + + tiny_node t1(g, hidden); + ASSERT(&tbb::flow::get<0>(t1.input_ports()) == &t1.f1, "f1 not bound to input port 0 in composite_node t1"); + ASSERT(&tbb::flow::get<0>(t1.output_ports()) == &t1.f2, "f2 not bound to output port 0 in composite_node t1"); + + test_input_ports_return_ref(t1); + test_output_ports_return_ref(t1); + + tiny_node t2(g, hidden); + ASSERT(&tbb::flow::input_port<0>(t2) == &t2.f1, "f1 not bound to input port 0 in composite_node t2"); + ASSERT(&tbb::flow::output_port<0>(t2) == &t2.f2, "f2 not bound to output port 0 in composite_node t2"); + + tbb::flow::function_node< int, int > f3( g, tbb::flow::unlimited, passthru_body() ); + tbb::flow::make_edge( f0, t ); + tbb::flow::make_edge( t, t1 ); + tbb::flow::make_edge( t1, t2 ); + tbb::flow::make_edge( t2 , f3 ); + tbb::flow::queue_node q(g); + tbb::flow::make_edge(f3, q); + f0.try_put(1); + g.wait_for_all(); + + int i, j =0; + q.try_get(i); + ASSERT( i == 1, "item did not go through graph"); + q.try_get(j); + ASSERT( !j, "unexpected item in graph"); + g.wait_for_all(); + + tbb::flow::remove_edge(f3, q); + tbb::flow::remove_edge(t2, f3); + tbb::flow::remove_edge(t1, t2); + + tbb::flow::make_edge( t1 , f3 ); + tbb::flow::make_edge(f3, q); + + f0.try_put(2); + g.wait_for_all(); + + q.try_get(i); + ASSERT( i == 2, "item did not go through graph after removal of edge"); + q.try_get(j); + ASSERT( !j, "unexpected item in graph after removal of edge"); + + return 0; +} + +class adder_node : public tbb::flow::composite_node< tbb::flow::tuple< int, int >, tbb::flow::tuple< int > > { +public: + tbb::flow::join_node< tbb::flow::tuple< int, int >, tbb::flow::queueing > j; + tbb::flow::function_node< tbb::flow::tuple< int, int >, int > f; +private: + typedef tbb::flow::composite_node< tbb::flow::tuple< int, int >, tbb::flow::tuple< int > > base_type; + + struct f_body { + int operator()( const tbb::flow::tuple< int, int > &t ) { + return tbb::flow::get<0>(t) + tbb::flow::get<1>(t); + } + }; + +public: + adder_node(tbb::flow::graph &g, bool hidden = false) : base_type(g), j(g), f(g, tbb::flow::unlimited, f_body() ) { + tbb::flow::make_edge( j, f ); + + base_type::set_external_ports(base_type::input_ports_type(tbb::flow::input_port<0>(j), tbb::flow::input_port<1>(j)), base_type::output_ports_type(f)); + + if (hidden) + base_type::add_nodes(j, f); + else + base_type::add_visible_nodes(j, f); + + } +}; + +struct square_body { int operator()(int v) { return v*v; } }; +struct cube_body { int operator()(int v) { return v*v*v; } }; +int adder_sum(int i) { + return (int)(pow(3*pow(i,3) + pow(i, 2),2)); +} +int test_adder(bool hidden = false) { + tbb::flow::graph g; + tbb::flow::function_node s(g, tbb::flow::unlimited, square_body()); + tbb::flow::function_node c(g, tbb::flow::unlimited, cube_body()); + tbb::flow::function_node p(g, tbb::flow::unlimited, passthru_body()); + + adder_node a0(g, hidden); + ASSERT(&tbb::flow::input_port<0>(a0) == &tbb::flow::input_port<0>(a0.j), "input_port 0 of j not bound to input port 0 in composite_node a0"); + ASSERT(&tbb::flow::input_port<1>(a0) == &tbb::flow::input_port<1>(a0.j), "input_port 1 of j not bound to input port 1 in composite_node a0"); + ASSERT(&tbb::flow::output_port<0>(a0) == &a0.f, "f not bound to output port 0 in composite_node a0"); + + adder_node a1(g, hidden); + ASSERT(&tbb::flow::get<0>(a0.input_ports()) == &tbb::flow::input_port<0>(a0.j), "input_port 0 of j not bound to input port 0 in composite_node a1"); + ASSERT(&tbb::flow::get<1>(a0.input_ports()) == &tbb::flow::input_port<1>(a0.j), "input_port1 of j not bound to input port 1 in composite_node a1"); + ASSERT(&tbb::flow::get<0>(a0.output_ports()) == &a0.f, "f not bound to output port 0 in composite_node a1"); + + adder_node a2(g, hidden); + ASSERT(&tbb::flow::input_port<0>(a2) == &tbb::flow::input_port<0>(a2.j), "input_port 0 of j not bound to input port 0 in composite_node a2"); + ASSERT(&tbb::flow::input_port<1>(a2) == &tbb::flow::input_port<1>(a2.j), "input_port 1 of j not bound to input port 1 in composite_node a2"); + ASSERT(&tbb::flow::output_port<0>(a2) == &a2.f, "f not bound to output port 0 in composite_node a2"); + + adder_node a3(g, hidden); + ASSERT(&tbb::flow::get<0>(a3.input_ports()) == &tbb::flow::input_port<0>(a3.j), "input_port 0 of j not bound to input port 0 in composite_node a3"); + ASSERT(&tbb::flow::get<1>(a3.input_ports()) == &tbb::flow::input_port<1>(a3.j), "input_port1 of j not bound to input port 1 in composite_node a3"); + ASSERT(&tbb::flow::get<0>(a3.output_ports()) == &a3.f, "f not bound to output port 0 in composite_node a3"); + + tbb::flow::function_node s2(g, tbb::flow::unlimited, square_body()); + tbb::flow::queue_node q(g); + + tbb::flow::make_edge( s, tbb::flow::input_port<0>(a0) ); + tbb::flow::make_edge( c, tbb::flow::input_port<1>(a0) ); + + tbb::flow::make_edge( c, tbb::flow::input_port<0>(a1) ); + tbb::flow::make_edge( c, tbb::flow::input_port<1>(a1) ); + + tbb::flow::make_edge( tbb::flow::output_port<0>(a0), tbb::flow::input_port<0>(a2) ); + tbb::flow::make_edge( tbb::flow::output_port<0>(a1), tbb::flow::input_port<1>(a2) ); + + tbb::flow::make_edge( tbb::flow::output_port<0>(a2), s2 ); + tbb::flow::make_edge( s2, q ); + + int sum_total=0; + int result=0; + for ( int i = 1; i < 4; ++i ) { + s.try_put(i); + c.try_put(i); + sum_total += adder_sum(i); + g.wait_for_all(); + } + + int j; + for ( int i = 1; i < 4; ++i ) { + q.try_get(j); + result += j; + } + g.wait_for_all(); + ASSERT(result == sum_total, "the sum from the graph does not match the calculated value"); + + tbb::flow::remove_edge(s2, q); + tbb::flow::remove_edge( a2, s2 ); + tbb::flow::make_edge( a0, a3 ); + tbb::flow::make_edge( a1, tbb::flow::input_port<1>(a3) ); + tbb::flow::make_edge( a3, s2 ); + tbb::flow::make_edge( s2, q ); + + sum_total=0; + result=0; + for ( int i = 10; i < 20; ++i ) { + s.try_put(i); + c.try_put(i); + sum_total += adder_sum(i); + g.wait_for_all(); + } + + for ( int i = 10; i < 20; ++i ) { + q.try_get(j); + result += j; + } + g.wait_for_all(); + ASSERT(result == sum_total, "the new sum after the replacement of the nodes does not match the calculated value"); + + return 0; +} + +/* + outer composite node (outer_node) + |-------------------------------------------------------------------| + | | + | |------------------| |------------------| |------------------| | + |---------------------| |--| inner composite | /| inner composite | /| inner composite | | |-------------------| + |broadcast node(input)|/| | node |/ | node |/ | node |-+-| queue node(output)| + |---------------------|\| |(inner_node1) |\ | (inner_node2) |\ | (inner_node3) | | |-------------------| + |--| | \| | \| | | + | |------------------| |------------------| |------------------| | + | | + |-------------------------------------------------------------------| + +*/ +int test_nested_adder(bool hidden=false) { + tbb::flow::graph g; + tbb::flow::composite_node, tbb::flow::tuple > outer_node(g); + typedef tbb::flow::composite_node, tbb::flow::tuple > base_type; + tbb::flow::broadcast_node input(g); + tbb::flow::queue_node output(g); + + adder_node inner_node1(g, hidden); + adder_node inner_node2(g, hidden); + adder_node inner_node3(g, hidden); + + outer_node.set_external_ports(base_type::input_ports_type(tbb::flow::input_port<0>(inner_node1), tbb::flow::input_port<1>(inner_node1)), base_type::output_ports_type(tbb::flow::output_port<0>(inner_node3))); + + ASSERT(&tbb::flow::input_port<0>(outer_node) == &tbb::flow::input_port<0>(inner_node1), "input port 0 of inner_node1 not bound to input port 0 in outer_node"); + ASSERT(&tbb::flow::input_port<1>(outer_node) == &tbb::flow::input_port<1>(inner_node1), "input port 1 of inner_node1 not bound to input port 1 in outer_node"); + ASSERT(&tbb::flow::output_port<0>(outer_node) == &tbb::flow::output_port<0>(inner_node3), "output port 0 of inner_node3 not bound to output port 0 in outer_node"); + + tbb::flow::make_edge(input, tbb::flow::input_port<0>(outer_node)/*inner_node1*/); + tbb::flow::make_edge(input, tbb::flow::input_port<1>(outer_node)/*inner_node1*/); + + tbb::flow::make_edge(inner_node1, tbb::flow::input_port<0>(inner_node2)); + tbb::flow::make_edge(inner_node1, tbb::flow::input_port<1>(inner_node2)); + + tbb::flow::make_edge(inner_node2, tbb::flow::input_port<0>(inner_node3)); + tbb::flow::make_edge(inner_node2, tbb::flow::input_port<1>(inner_node3)); + + tbb::flow::make_edge(outer_node/*inner_node3*/, output); + + if(hidden) + outer_node.add_nodes(inner_node1, inner_node2, inner_node3); + else + outer_node.add_visible_nodes(inner_node1, inner_node2, inner_node3); + + int out; + for (int i = 1; i < 200000; ++i) { + input.try_put(i); + g.wait_for_all(); + output.try_get(out); + ASSERT(tbb::flow::output_port<0>(outer_node).try_get(out) == output.try_get(out), "output from outer_node does not match output from graph"); + ASSERT(out == 8*i, "output from outer_node not correct"); + } + g.wait_for_all(); + + return 0; +} + +template< typename T > +class prefix_node : public tbb::flow::composite_node< tbb::flow::tuple< T, T, T, T, T >, tbb::flow::tuple< T, T, T, T, T > > { + typedef tbb::flow::tuple< T, T, T, T, T > my_tuple_t; +public: + tbb::flow::join_node< my_tuple_t, tbb::flow::queueing > j; + tbb::flow::split_node< my_tuple_t > s; +private: + tbb::flow::function_node< my_tuple_t, my_tuple_t > f; + typedef tbb::flow::composite_node< my_tuple_t, my_tuple_t > base_type; + + struct f_body { + my_tuple_t operator()( const my_tuple_t &t ) { + return my_tuple_t( tbb::flow::get<0>(t), + tbb::flow::get<0>(t) + tbb::flow::get<1>(t), + tbb::flow::get<0>(t) + tbb::flow::get<1>(t) + tbb::flow::get<2>(t), + tbb::flow::get<0>(t) + tbb::flow::get<1>(t) + tbb::flow::get<2>(t) + tbb::flow::get<3>(t), + tbb::flow::get<0>(t) + tbb::flow::get<1>(t) + tbb::flow::get<2>(t) + tbb::flow::get<3>(t) + tbb::flow::get<4>(t) ); + } + }; + +public: + prefix_node(tbb::flow::graph &g, bool hidden = false ) : base_type(g), j(g), s(g), f(g, tbb::flow::serial, f_body() ) { + tbb::flow::make_edge( j, f ); + tbb::flow::make_edge( f, s ); + + typename base_type::input_ports_type input_tuple(tbb::flow::input_port<0>(j), tbb::flow::input_port<1>(j), tbb::flow::input_port<2>(j), tbb::flow::input_port<3>(j), tbb::flow::input_port<4>(j)); + + typename base_type::output_ports_type output_tuple(tbb::flow::output_port<0>(s), tbb::flow::output_port<1>(s), tbb::flow::output_port<2>(s), tbb::flow::output_port<3>(s), tbb::flow::output_port<4>(s)); + + base_type::set_external_ports(input_tuple, output_tuple); + + if(hidden) + base_type::add_nodes(j,s,f); + else + base_type::add_visible_nodes(j,s,f); + + } +}; + +int test_prefix(bool hidden = false) { + tbb::flow::graph g; + prefix_node p(g, hidden); + + ASSERT(&tbb::flow::get<0>(p.input_ports()) == &tbb::flow::input_port<0>(p.j), "input port 0 of j is not bound to input port 0 of composite node p"); + ASSERT(&tbb::flow::input_port<1>(p.j) == &tbb::flow::input_port<1>(p.j), "input port 1 of j is not bound to input port 1 of composite node p"); + ASSERT(&tbb::flow::get<2>(p.input_ports()) == &tbb::flow::input_port<2>(p.j), "input port 2 of j is not bound to input port 2 of composite node p"); + ASSERT(&tbb::flow::input_port<3>(p.j) == &tbb::flow::input_port<3>(p.j), "input port 3 of j is not bound to input port 3 of composite node p"); + ASSERT(&tbb::flow::get<4>(p.input_ports()) == &tbb::flow::input_port<4>(p.j), "input port 4 of j is not bound to input port 4 of composite node p"); + + + ASSERT(&tbb::flow::get<0>(p.output_ports()) == &tbb::flow::output_port<0>(p.s), "output port 0 of s is not bound to output port 0 of composite node p"); + ASSERT(&tbb::flow::output_port<1>(p.s) == &tbb::flow::output_port<1>(p.s), "output port 1 of s is not bound to output port 1 of composite node p"); + ASSERT(&tbb::flow::get<2>(p.output_ports()) == &tbb::flow::output_port<2>(p.s), "output port 2 of s is not bound to output port 2 of composite node p"); + ASSERT(&tbb::flow::output_port<3>(p.s) == &tbb::flow::output_port<3>(p.s), "output port 3 of s is not bound to output port 3 of composite node p"); + ASSERT(&tbb::flow::get<4>(p.output_ports()) == &tbb::flow::output_port<4>(p.s), "output port 4 of s is not bound to output port 4 of composite node p"); + + std::vector< tbb::flow::queue_node > v( 5, tbb::flow::queue_node(g) ); + tbb::flow::make_edge( tbb::flow::output_port<0>(p), v[0] ); + tbb::flow::make_edge( tbb::flow::output_port<1>(p), v[1] ); + tbb::flow::make_edge( tbb::flow::output_port<2>(p), v[2] ); + tbb::flow::make_edge( tbb::flow::output_port<3>(p), v[3] ); + tbb::flow::make_edge( tbb::flow::output_port<4>(p), v[4] ); + + for( double offset = 1; offset < 10000; offset *= 10 ) { + tbb::flow::input_port<0>(p).try_put( offset ); + tbb::flow::input_port<1>(p).try_put( offset + 1 ); + tbb::flow::input_port<2>(p).try_put( offset + 2 ); + tbb::flow::input_port<3>(p).try_put( offset + 3 ); + tbb::flow::input_port<4>(p).try_put( offset + 4 ); + } + g.wait_for_all(); + + double x; + while ( v[0].try_get(x) ) { + g.wait_for_all(); + for ( int i = 1; i < 5; ++i ) { + v[i].try_get(x); + g.wait_for_all(); + } + } + return 0; +} + +struct input_only_output_only_seq { + int operator()(int i){ return (i + 3) / 4 - 1;} +}; + +void input_only_output_only_composite(bool hidden) { + tbb::flow::graph g; +#if TBB_PREVIEW_FLOW_GRAPH_TRACE + tbb::flow::composite_node, tbb::flow::tuple > input_output(g, "test_name"); +#else + tbb::flow::composite_node, tbb::flow::tuple > input_output(g); +#endif + typedef tbb::flow::composite_node, tbb::flow::tuple<> > input_only_composite; + typedef tbb::flow::composite_node, tbb::flow::tuple > output_only_composite; + typedef tbb::flow::source_node src_type; + typedef tbb::flow::queue_node q_type; + typedef tbb::flow::function_node f_type; + typedef tbb::flow::sequencer_node sequencer_type; + + int num = 0; + int finish=1000; + int step = 4; + + input_only_composite a_in(g); + output_only_composite a_out(g); + + src_type src(g, src_body(finish, step), false); + q_type que(g); + f_type f(g, 1, passthru_body()); + + // Sequencer_node is needed, because serial function_node guarantees only serial body execution, + // not a sequential order of messages dispatch + sequencer_type seq(g, input_only_output_only_seq()); + + tbb::flow::tuple input_tuple(f); + a_in.set_external_ports(input_tuple); + ASSERT(&tbb::flow::get<0>(a_in.input_ports()) == &f, "f not bound to input port 0 in composite_node a_in"); + + tbb::flow::tuple output_tuple(src); + a_out.set_external_ports(output_tuple); + ASSERT(&tbb::flow::get<0>(a_out.output_ports()) == &src, "src not bound to output port 0 in composite_node a_out"); + + if(hidden) { + a_in.add_nodes(f, seq, que); + a_out.add_nodes(src); + } else { + a_in.add_visible_nodes(f, seq, que); + a_out.add_visible_nodes(src); + } + + tbb::flow::make_edge(a_out, a_in); + tbb::flow::make_edge(f, seq); + tbb::flow::make_edge(seq, que); + src.activate(); + g.wait_for_all(); + + for(int i = 1; i,std::allocator> > MyAllocator; + +template +inline void CheckAllocator(typename Table::allocator_type& a, size_t expected_allocs, size_t expected_frees, + bool exact = true) { + if(exact) { + ASSERT( a.allocations == expected_allocs, NULL); ASSERT( a.frees == expected_frees, NULL); + } else { + ASSERT( a.allocations >= expected_allocs, NULL); ASSERT( a.frees >= expected_frees, NULL); + ASSERT( a.allocations - a.frees == expected_allocs - expected_frees, NULL ); + } +} + +// Check that only dummy node allocated if table is empty +// Specialize this function for custom container, if it node allocation size > 1 +#define CheckEmptyContainerAllocatorE(t,a,f) CheckEmptyContainerAllocator(t,a,f,true,__LINE__) +#define CheckEmptyContainerAllocatorA(t,a,f) CheckEmptyContainerAllocator(t,a,f,false,__LINE__) +template +inline void CheckEmptyContainerAllocator(MyTable &table, size_t expected_allocs, size_t expected_frees, bool exact = true, int line = 0); + +template +struct strip_const { typedef T type; }; + +template +struct strip_const { typedef T type; }; + +// value generator for map +template > +struct ValueFactory { + typedef typename strip_const::type Kstrip; + static V make(const K &value) { return V(value, value); } + static Kstrip key(const V &value) { return value.first; } + static Kstrip get(const V &value) { return (Kstrip)value.second; } + template< typename U > + static U convert(const V &value) { return U(value.second); } +}; + +// generator for set +template +struct ValueFactory { + static T make(const T &value) { return value; } + static T key(const T &value) { return value; } + static T get(const T &value) { return value; } + template< typename U > + static U convert(const T &value) { return U(value); } +}; + +template +struct Value : ValueFactory { + template + static bool compare( const typename T::iterator& it, U val ) { + return (Value::template convert(*it) == val); + } +}; + +template +void check_value_state(/* typename do_check_element_state =*/ tbb::internal::true_type, T const& t, const char* filename, int line ) +{ + ASSERT_CUSTOM(is_state_f()(t), "", filename, line); +} + +template +void check_value_state(/* typename do_check_element_state =*/ tbb::internal::false_type, T const&, const char* , int ) {/*do nothing*/} + +#define ASSERT_VALUE_STATE(do_check_element_state,state,value) check_value_state(do_check_element_state,value,__FILE__,__LINE__) + +#if __TBB_CPP11_RVALUE_REF_PRESENT +template +void test_rvalue_insert(V v1, V v2) +{ + typedef T container_t; + + container_t cont; + + std::pair ins = cont.insert(Value::make(v1)); + ASSERT(ins.second == true && Value::get(*(ins.first)) == v1, "Element 1 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::MoveInitialized,*ins.first); + + typename container_t::iterator it2 = cont.insert(ins.first, Value::make(v2)); + ASSERT(Value::get(*(it2)) == v2, "Element 2 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::MoveInitialized,*it2); + +} +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +// The test does not use variadic templates, but emplace() does. + +namespace emplace_helpers { +template +std::pair call_emplace_impl(container_t& c, arg_t&& k, value_t *){ + // this is a set + return c.emplace(std::forward(k)); +} + +template +std::pair call_emplace_impl(container_t& c, arg_t&& k, std::pair *){ + // this is a map + return c.emplace(k, std::forward(k)); +} + +template +std::pair call_emplace(container_t& c, arg_t&& k){ + typename container_t::value_type * selector = NULL; + return call_emplace_impl(c, std::forward(k), selector); +} + +template +typename container_t::iterator call_emplace_hint_impl(container_t& c, typename container_t::const_iterator hint, arg_t&& k, value_t *){ + // this is a set + return c.emplace_hint(hint, std::forward(k)); +} + +template +typename container_t::iterator call_emplace_hint_impl(container_t& c, typename container_t::const_iterator hint, arg_t&& k, std::pair *){ + // this is a map + return c.emplace_hint(hint, k, std::forward(k)); +} + +template +typename container_t::iterator call_emplace_hint(container_t& c, typename container_t::const_iterator hint, arg_t&& k){ + typename container_t::value_type * selector = NULL; + return call_emplace_hint_impl(c, hint, std::forward(k), selector); +} +} +template +void test_emplace_insert(V v1, V v2){ + typedef T container_t; + container_t cont; + + std::pair ins = emplace_helpers::call_emplace(cont, v1); + ASSERT(ins.second == true && Value::compare(ins.first, v1), "Element 1 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::DirectInitialized,*ins.first); + + typename container_t::iterator it2 = emplace_helpers::call_emplace_hint(cont, ins.first, v2); + ASSERT(Value::compare(it2, v2), "Element 2 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::DirectInitialized,*it2); +} +#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +template +std::pair CheckRecursiveRange(RangeType range) { + std::pair sum(0, 0); // count, sum + for( Iterator i = range.begin(), e = range.end(); i != e; ++i ) { + ++sum.first; sum.second += Value::get(*i); + } + if( range.is_divisible() ) { + RangeType range2( range, tbb::split() ); + std::pair sum1 = CheckRecursiveRange( range ); + std::pair sum2 = CheckRecursiveRange( range2 ); + sum1.first += sum2.first; sum1.second += sum2.second; + ASSERT( sum == sum1, "Mismatched ranges after division"); + } + return sum; +} + +template +void SpecialMapTests( const char *str ){ + Map cont; + const Map &ccont( cont ); + + // mapped_type& operator[](const key_type& k); + cont[1] = 2; + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 1, "Concurrent container size incorrect" ); + ASSERT( cont[1] == 2, "Concurrent container value incorrect" ); + + // mapped_type& at( const key_type& k ); + // const mapped_type& at(const key_type& k) const; + ASSERT( cont.at( 1 ) == 2, "Concurrent container value incorrect" ); + ASSERT( ccont.at( 1 ) == 2, "Concurrent container value incorrect" ); + + // iterator find(const key_type& k); + typename Map::iterator it = cont.find( 1 ); + ASSERT( it != cont.end( ) && Value::get( *(it) ) == 2, "Element with key 1 not properly found" ); + cont.unsafe_erase( it ); + + it = cont.find( 1 ); + ASSERT( it == cont.end( ), "Element with key 1 not properly erased" ); + REMARK( "passed -- specialized %s tests\n", str ); +} + +template +void CheckMultiMap(MultiMap &m, int *targets, int tcount, int key) { + std::vector vfound(tcount,false); + std::pair range = m.equal_range( key ); + for(typename MultiMap::iterator it = range.first; it != range.second; ++it) { + bool found = false; + for( int i = 0; i < tcount; ++i) { + if((*it).second == targets[i]) { + if(!vfound[i]) { // we can insert duplicate values + vfound[i] = found = true; + break; + } + } + } + // just in case an extra value in equal_range... + ASSERT(found, "extra value from equal range"); + } + for(int i = 0; i < tcount; ++i) ASSERT(vfound[i], "missing value"); +} + +template +void SpecialMultiMapTests( const char *str ){ + int one_values[] = { 7, 2, 13, 23, 13 }; + int zero_values[] = { 4, 9, 13, 29, 42, 111}; + int n_zero_values = sizeof(zero_values) / sizeof(int); + int n_one_values = sizeof(one_values) / sizeof(int); + MultiMap cont; + const MultiMap &ccont( cont ); + // mapped_type& operator[](const key_type& k); + cont.insert( std::make_pair( 1, one_values[0] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 1, "Concurrent container size incorrect" ); + ASSERT( (*(cont.begin( ))).second == one_values[0], "Concurrent container value incorrect" ); + ASSERT( (*(cont.equal_range( 1 )).first).second == one_values[0], "Improper value from equal_range" ); + ASSERT( (cont.equal_range( 1 )).second == cont.end( ), "Improper iterator from equal_range" ); + + cont.insert( std::make_pair( 1, one_values[1] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 2, "Concurrent container size incorrect" ); + CheckMultiMap(cont, one_values, 2, 1); + + // insert the other {1,x} values + for( int i = 2; i < n_one_values; ++i ) { + cont.insert( std::make_pair( 1, one_values[i] ) ); + } + + CheckMultiMap(cont, one_values, n_one_values, 1); + ASSERT( (cont.equal_range( 1 )).second == cont.end( ), "Improper iterator from equal_range" ); + + cont.insert( std::make_pair( 0, zero_values[0] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == (size_t)(n_one_values+1), "Concurrent container size incorrect" ); + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, 1, 0); + ASSERT( (*(cont.begin( ))).second == zero_values[0], "Concurrent container value incorrect" ); + // insert the rest of the zero values + for( int i = 1; i < n_zero_values; ++i) { + cont.insert( std::make_pair( 0, zero_values[i] ) ); + } + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, n_zero_values, 0); + + // clear, reinsert interleaved + cont.clear(); + int bigger_num = ( n_one_values > n_zero_values ) ? n_one_values : n_zero_values; + for( int i = 0; i < bigger_num; ++i ) { + if(i < n_one_values) cont.insert( std::make_pair( 1, one_values[i] ) ); + if(i < n_zero_values) cont.insert( std::make_pair( 0, zero_values[i] ) ); + } + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, n_zero_values, 0); + + + REMARK( "passed -- specialized %s tests\n", str ); +} + +template +struct SpecialTests { + static void Test(const char *str) {REMARK("skipped -- specialized %s tests\n", str);} +}; + + + +#if __TBB_RANGE_BASED_FOR_PRESENT +#include "test_range_based_for.h" + +template +void TestRangeBasedFor() { + using namespace range_based_for_support_tests; + + REMARK( "testing range based for loop compatibility \n" ); + Container cont; + const int sequence_length = 100; + for ( int i = 1; i <= sequence_length; ++i ) { + cont.insert( Value::make(i) ); + } + + ASSERT( range_based_for_accumulate( cont, unified_summer(), 0 ) == + gauss_summ_of_int_sequence( sequence_length ), + "incorrect accumulated value generated via range based for ?" ); +} +#endif /* __TBB_RANGE_BASED_FOR_PRESENT */ + +#if __TBB_INITIALIZER_LISTS_PRESENT +// Required by test_initializer_list.h +template +bool equal_containers(container_type const& lhs, container_type const& rhs) { + if ( lhs.size() != rhs.size() ) { + return false; + } + return std::equal( lhs.begin(), lhs.end(), rhs.begin(), Harness::IsEqual() ); +} + +#include "test_initializer_list.h" + +template +void TestInitList( std::initializer_list il ) { + using namespace initializer_list_support_tests; + REMARK("testing initializer_list methods \n"); + + TestInitListSupportWithoutAssign(il); + TestInitListSupportWithoutAssign( il ); + +#if __TBB_ICC_EMPTY_INIT_LIST_TESTS_BROKEN + REPORT( "Known issue: TestInitListSupportWithoutAssign with an empty initializer list is skipped.\n"); +#else + TestInitListSupportWithoutAssign( {} ); + TestInitListSupportWithoutAssign( {} ); +#endif +} +#endif //if __TBB_INITIALIZER_LISTS_PRESENT + +template +void test_basic_common(const char * str, do_check_element_state) +{ + T cont; + const T &ccont(cont); + CheckEmptyContainerAllocatorE(cont, 1, 0); // one dummy is always allocated + // bool empty() const; + ASSERT(ccont.empty(), "Concurrent container is not empty after construction"); + + // size_type size() const; + ASSERT(ccont.size() == 0, "Concurrent container is not empty after construction"); + + // size_type max_size() const; + ASSERT(ccont.max_size() > 0, "Concurrent container max size is invalid"); + + //iterator begin(); + //iterator end(); + ASSERT(cont.begin() == cont.end(), "Concurrent container iterators are invalid after construction"); + ASSERT(ccont.begin() == ccont.end(), "Concurrent container iterators are invalid after construction"); + ASSERT(cont.cbegin() == cont.cend(), "Concurrent container iterators are invalid after construction"); + + //std::pair insert(const value_type& obj); + std::pair ins = cont.insert(Value::make(1)); + ASSERT(ins.second == true && Value::get(*(ins.first)) == 1, "Element 1 has not been inserted properly"); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + test_rvalue_insert(1,2); +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + test_emplace_insert(1,2); +#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + + // bool empty() const; + ASSERT(!ccont.empty(), "Concurrent container is empty after adding an element"); + + // size_type size() const; + ASSERT(ccont.size() == 1, "Concurrent container size is incorrect"); + + std::pair ins2 = cont.insert(Value::make(1)); + + if (T::allow_multimapping) + { + // std::pair insert(const value_type& obj); + ASSERT(ins2.second == true && Value::get(*(ins2.first)) == 1, "Element 1 has not been inserted properly"); + + // size_type size() const; + ASSERT(ccont.size() == 2, "Concurrent container size is incorrect"); + + // size_type count(const key_type& k) const; + ASSERT(ccont.count(1) == 2, "Concurrent container count(1) is incorrect"); + // std::pair equal_range(const key_type& k); + std::pair range = cont.equal_range(1); + typename T::iterator it = range.first; + ASSERT(it != cont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + unsigned int count = 0; + for (; it != range.second; it++) + { + count++; + ASSERT(Value::get(*it) == 1, "Element 1 has not been found properly"); + } + + ASSERT(count == 2, "Range doesn't have the right number of elements"); + } + else + { + // std::pair insert(const value_type& obj); + ASSERT(ins2.second == false && ins2.first == ins.first, "Element 1 should not be re-inserted"); + + // size_type size() const; + ASSERT(ccont.size() == 1, "Concurrent container size is incorrect"); + + // size_type count(const key_type& k) const; + ASSERT(ccont.count(1) == 1, "Concurrent container count(1) is incorrect"); + + // std::pair equal_range(const key_type& k) const; + // std::pair equal_range(const key_type& k); + std::pair range = cont.equal_range(1); + typename T::iterator it = range.first; + ASSERT(it != cont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + } + + // const_iterator find(const key_type& k) const; + // iterator find(const key_type& k); + typename T::iterator it = cont.find(1); + ASSERT(it != cont.end() && Value::get(*(it)) == 1, "Element 1 has not been found properly"); + ASSERT(ccont.find(1) == it, "Element 1 has not been found properly"); + + // Will be implemented in unordered containers later +#if !__TBB_UNORDERED_TEST + //bool contains(const key_type&k) const + ASSERT(cont.contains(1), "contains() cannot detect existing element"); + ASSERT(!cont.contains(0), "contains() detect not existing element"); +#endif /*__TBB_UNORDERED_TEST*/ + + // iterator insert(const_iterator hint, const value_type& obj); + typename T::iterator it2 = cont.insert(ins.first, Value::make(2)); + ASSERT(Value::get(*it2) == 2, "Element 2 has not been inserted properly"); + + // T(const T& _Umap) + T newcont = ccont; + ASSERT(T::allow_multimapping ? (newcont.size() == 3) : (newcont.size() == 2), "Copy construction has not copied the elements properly"); + + // this functionality not implemented yet + // size_type unsafe_erase(const key_type& k); + typename T::size_type size = cont.unsafe_erase(1); + ASSERT(T::allow_multimapping ? (size == 2) : (size == 1), "Erase has not removed the right number of elements"); + + // iterator unsafe_erase(const_iterator position); + typename T::iterator it4 = cont.unsafe_erase(cont.find(2)); + ASSERT(it4 == cont.end() && cont.size() == 0, "Erase has not removed the last element properly"); + + // template void insert(InputIterator first, InputIterator last); + cont.insert(newcont.begin(), newcont.end()); + ASSERT(T::allow_multimapping ? (cont.size() == 3) : (cont.size() == 2), "Range insert has not copied the elements properly"); + + // this functionality not implemented yet + // iterator unsafe_erase(const_iterator first, const_iterator last); + std::pair range2 = newcont.equal_range(1); + newcont.unsafe_erase(range2.first, range2.second); + ASSERT(newcont.size() == 1, "Range erase has not erased the elements properly"); + + // void clear(); + newcont.clear(); + ASSERT(newcont.begin() == newcont.end() && newcont.size() == 0, "Clear has not cleared the container"); + +#if __TBB_INITIALIZER_LISTS_PRESENT +#if __TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN + REPORT("Known issue: the test for insert with initializer_list is skipped.\n"); +#else + // void insert(const std::initializer_list &il); + newcont.insert( { Value::make( 1 ), Value::make( 2 ), Value::make( 1 ) } ); + if (T::allow_multimapping) { + ASSERT(newcont.size() == 3, "Concurrent container size is incorrect"); + ASSERT(newcont.count(1) == 2, "Concurrent container count(1) is incorrect"); + ASSERT(newcont.count(2) == 1, "Concurrent container count(2) is incorrect"); + std::pair range = cont.equal_range(1); + it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + unsigned int count = 0; + for (; it != range.second; it++) { + count++; + ASSERT(Value::get(*it) == 1, "Element 1 has not been found properly"); + } + ASSERT(count == 2, "Range doesn't have the right number of elements"); + range = newcont.equal_range(2); it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 2, "Element 2 has not been found properly"); + count = 0; + for (; it != range.second; it++) { + count++; + ASSERT(Value::get(*it) == 2, "Element 2 has not been found properly"); + } + ASSERT(count == 1, "Range doesn't have the right number of elements"); + } else { + ASSERT(newcont.size() == 2, "Concurrent container size is incorrect"); + ASSERT(newcont.count(1) == 1, "Concurrent container count(1) is incorrect"); + ASSERT(newcont.count(2) == 1, "Concurrent container count(2) is incorrect"); + std::pair range = newcont.equal_range(1); + it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + range = newcont.equal_range(2); it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 2, "Element 2 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + } +#endif /* __TBB_CPP11_INIT_LIST_TEMP_OBJS_COMPILATION_BROKEN */ +#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ + + // T& operator=(const T& _Umap) + newcont = ccont; + ASSERT(T::allow_multimapping ? (newcont.size() == 3) : (newcont.size() == 2), "Assignment operator has not copied the elements properly"); + + REMARK("passed -- basic %s tests\n", str); + +#if defined (VERBOSE) + REMARK("container dump debug:\n"); + cont._Dump(); + REMARK("container dump release:\n"); + cont.dump(); + REMARK("\n"); +#endif + + cont.clear(); + CheckEmptyContainerAllocatorA(cont, 1, 0); // one dummy is always allocated + for (int i = 0; i < 256; i++) + { + std::pair ins3 = cont.insert(Value::make(i)); + ASSERT(ins3.second == true && Value::get(*(ins3.first)) == i, "Element 1 has not been inserted properly"); + } + ASSERT(cont.size() == 256, "Wrong number of elements have been inserted"); + ASSERT((256 == CheckRecursiveRange(cont.range()).first), NULL); + ASSERT((256 == CheckRecursiveRange(ccont.range()).first), NULL); + + // void swap(T&); + cont.swap(newcont); + ASSERT(newcont.size() == 256, "Wrong number of elements after swap"); + ASSERT(newcont.count(200) == 1, "Element with key 200 is not present after swap"); + ASSERT(newcont.count(16) == 1, "Element with key 16 is not present after swap"); + ASSERT(newcont.count(99) == 1, "Element with key 99 is not present after swap"); + ASSERT(T::allow_multimapping ? (cont.size() == 3) : (cont.size() == 2), "Assignment operator has not copied the elements properly"); + + // Need to be enabled + SpecialTests::Test(str); +} + +template +void test_basic_common(const char * str){ + test_basic_common(str, tbb::internal::false_type()); +} + +void test_machine() { + ASSERT(__TBB_ReverseByte(0)==0, NULL ); + ASSERT(__TBB_ReverseByte(1)==0x80, NULL ); + ASSERT(__TBB_ReverseByte(0xFE)==0x7F, NULL ); + ASSERT(__TBB_ReverseByte(0xFF)==0xFF, NULL ); +} + +template +class FillTable: NoAssign { + T &table; + const int items; + bool my_asymptotic; + typedef std::pair pairIB; +public: + FillTable(T &t, int i, bool asymptotic) : table(t), items(i), my_asymptotic(asymptotic) { + ASSERT( !(items&1) && items > 100, NULL); + } + void operator()(int threadn) const { + if( threadn == 0 ) { // Fill even keys forward (single thread) + bool last_inserted = true; + for( int i = 0; i < items; i+=2 ) { + pairIB pib = table.insert(Value::make(my_asymptotic?1:i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic?1:i), "Element not properly inserted"); + ASSERT( last_inserted || !pib.second, "Previous key was not inserted but this is inserted" ); + last_inserted = pib.second; + } + } else if( threadn == 1 ) { // Fill even keys backward (single thread) + bool last_inserted = true; + for( int i = items-2; i >= 0; i-=2 ) { + pairIB pib = table.insert(Value::make(my_asymptotic?1:i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic?1:i), "Element not properly inserted"); + ASSERT( last_inserted || !pib.second, "Previous key was not inserted but this is inserted" ); + last_inserted = pib.second; + } + } else if( !(threadn&1) ) { // Fill odd keys forward (multiple threads) + for( int i = 1; i < items; i+=2 ) +#if __TBB_INITIALIZER_LISTS_PRESENT && !__TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN + if ( i % 32 == 1 && i + 6 < items ) { + if (my_asymptotic) { + table.insert({ Value::make(1), Value::make(1), Value::make(1) }); + ASSERT(Value::get(*table.find(1)) == 1, "Element not properly inserted"); + } + else { + table.insert({ Value::make(i), Value::make(i + 2), Value::make(i + 4) }); + ASSERT(Value::get(*table.find(i)) == i, "Element not properly inserted"); + ASSERT(Value::get(*table.find(i + 2)) == i + 2, "Element not properly inserted"); + ASSERT(Value::get(*table.find(i + 4)) == i + 4, "Element not properly inserted"); + } + i += 4; + } else +#endif + { + pairIB pib = table.insert(Value::make(my_asymptotic ? 1 : i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic ? 1 : i), "Element not properly inserted"); + } + } else { // Check odd keys backward (multiple threads) + if (!my_asymptotic) { + bool last_found = false; + for( int i = items-1; i >= 0; i-=2 ) { + typename T::iterator it = table.find(i); + if( it != table.end() ) { // found + ASSERT(Value::get(*it) == i, "Element not properly inserted"); + last_found = true; + } else { + ASSERT( !last_found, "Previous key was found but this is not" ); + } + } + } + } + } +}; + +typedef tbb::atomic AtomicByte; + +template +struct ParallelTraverseBody: NoAssign { + const int n; + AtomicByte* const array; + ParallelTraverseBody( AtomicByte an_array[], int a_n ) : + n(a_n), array(an_array) + {} + void operator()( const RangeType& range ) const { + for( typename RangeType::iterator i = range.begin(); i!=range.end(); ++i ) { + int k = static_cast(Value::key(*i)); + ASSERT( k == Value::get(*i), NULL ); + ASSERT( 0<=k && k +class CheckTable: NoAssign { + T &table; +public: + CheckTable(T &t) : NoAssign(), table(t) {} + void operator()(int i) const { + int c = (int)table.count( i ); + ASSERT( c, "must exist" ); + } +}; + +template +void test_concurrent_common(const char *tablename, bool asymptotic = false) { +#if TBB_USE_ASSERT + int items = 2000; +#else + int items = 20000; +#endif + int nItemsInserted = 0; + int nThreads = 0; +#if __TBB_UNORDERED_TEST + T table(items/1000); +#else + T table; +#endif + #if __bgp__ + nThreads = 6; + #else + nThreads = 16; + #endif + if(T::allow_multimapping) { + // even passes (threads 0 & 1) put N/2 items each + // odd passes (threads > 1) put N/2 if thread is odd, else checks if even. + items = 4*items / (nThreads + 2); // approximately same number of items inserted. + nItemsInserted = items + (nThreads-2) * items / 4; + } + else { + nItemsInserted = items; + } + REMARK("%s items == %d\n", tablename, items); + tbb::tick_count t0 = tbb::tick_count::now(); + NativeParallelFor( nThreads, FillTable(table, items, asymptotic) ); + tbb::tick_count t1 = tbb::tick_count::now(); + REMARK( "time for filling '%s' by %d items = %g\n", tablename, table.size(), (t1-t0).seconds() ); + ASSERT( int(table.size()) == nItemsInserted, NULL); + + if(!asymptotic) { + AtomicByte* array = new AtomicByte[items]; + memset( static_cast(array), 0, items*sizeof(AtomicByte) ); + + typename T::range_type r = table.range(); + std::pair p = CheckRecursiveRange(r); + ASSERT((nItemsInserted == p.first), NULL); + tbb::parallel_for( r, ParallelTraverseBody( array, items )); + CheckRange( array, items, T::allow_multimapping, (nThreads - 1)/2 ); + + const T &const_table = table; + memset( static_cast(array), 0, items*sizeof(AtomicByte) ); + typename T::const_range_type cr = const_table.range(); + ASSERT((nItemsInserted == CheckRecursiveRange(cr).first), NULL); + tbb::parallel_for( cr, ParallelTraverseBody( array, items )); + CheckRange( array, items, T::allow_multimapping, (nThreads - 1) / 2 ); + delete[] array; + + tbb::parallel_for( 0, items, CheckTable( table ) ); + } + + table.clear(); + CheckEmptyContainerAllocatorA(table, items+1, items); // one dummy is always allocated + +} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include "test_container_move_support.h" + +template +void test_rvalue_ref_support(const char* container_name){ + TestMoveConstructor(); + TestMoveAssignOperator(); +#if TBB_USE_EXCEPTIONS + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorMemoryFailure(); + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorExceptionInElementCtor(); +#endif //TBB_USE_EXCEPTIONS + REMARK("passed -- %s move support tests\n", container_name); +} +#endif //__TBB_CPP11_RVALUE_REF_PRESENT + +namespace test_select_size_t_constant{ + __TBB_STATIC_ASSERT((tbb::internal::select_size_t_constant<1234,1234>::value == 1234),"select_size_t_constant::value is not compile time constant"); +// There will be two constant used in the test 32 bit and 64 bit one. +// The 64 bit constant should chosen so that it 32 bit halves adds up to the 32 bit one ( first constant used in the test). +// % ~0U is used to sum up 32bit halves of the 64 constant. ("% ~0U" essentially adds the 32-bit "digits", like "%9" adds +// the digits (modulo 9) of a number in base 10). +// So iff select_size_t_constant is correct result of the calculation below will be same on both 32bit and 64bit platforms. + __TBB_STATIC_ASSERT((tbb::internal::select_size_t_constant<0x12345678U,0x091A2B3C091A2B3CULL>::value % ~0U == 0x12345678U), + "select_size_t_constant have chosen the wrong constant"); +} + +#if __TBB_CPP11_SMART_POINTERS_PRESENT +// For the sake of simplified testing, make unique_ptr implicitly convertible to/from the pointer +namespace test { + template + class unique_ptr : public std::unique_ptr { + public: + typedef typename std::unique_ptr::pointer pointer; + unique_ptr( pointer p ) : std::unique_ptr(p) {} + operator pointer() const { return this->get(); } + }; +} +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ + +#include +#include +#include + +template +class TestRange : NoAssign { + const std::list &my_lst; + std::vector< tbb::atomic > &my_marks; +public: + TestRange( const std::list &lst, std::vector< tbb::atomic > &marks ) : my_lst( lst ), my_marks( marks ) { + std::fill( my_marks.begin(), my_marks.end(), false ); + } + template + void operator()( const Range &r ) const { doTestRange( r.begin(), r.end() ); } + template + void doTestRange( Iterator i, Iterator j ) const { + for ( Iterator it = i; it != j; ) { + Iterator prev_it = it++; + typename std::list::const_iterator it2 = std::search( my_lst.begin(), my_lst.end(), prev_it, it, Harness::IsEqual() ); + ASSERT( it2 != my_lst.end(), NULL ); + typename std::list::difference_type dist = std::distance( my_lst.begin( ), it2 ); + ASSERT( !my_marks[dist], NULL ); + my_marks[dist] = true; + } + } +}; + +// The helper to call a function only when a doCall == true. +template struct CallIf { + template void operator() ( FuncType func ) const { func(); } +}; +template <> struct CallIf { + template void operator()( FuncType ) const {} +}; + +template +class TestOperatorSquareBrackets : NoAssign { + typedef typename Table::value_type ValueType; + Table &my_c; + const ValueType &my_value; +public: + TestOperatorSquareBrackets( Table &c, const ValueType &value ) : my_c( c ), my_value( value ) {} + void operator()() const { + ASSERT( Harness::IsEqual()(my_c[my_value.first], my_value.second), NULL ); + } +}; + +template +void TestMapSpecificMethodsImpl(Table &c, const Value &value){ + CallIf()(TestOperatorSquareBrackets( c, value )); + ASSERT( Harness::IsEqual()(c.at( value.first ), value.second), NULL ); + const Table &constC = c; + ASSERT( Harness::IsEqual()(constC.at( value.first ), value.second), NULL ); +} + +// do nothing for common case +template +void TestMapSpecificMethods( Table&, const Value& ) {} + +template +class CheckValue : NoAssign { + Table &my_c; +public: + CheckValue( Table &c ) : my_c( c ) {} + void operator()( const typename Table::value_type &value ) { + typedef typename Table::iterator Iterator; + typedef typename Table::const_iterator ConstIterator; + const Table &constC = my_c; + ASSERT( my_c.count( Value
::key( value ) ) == 1, NULL ); + // find + ASSERT( Harness::IsEqual()(*my_c.find( Value
::key( value ) ), value), NULL ); + ASSERT( Harness::IsEqual()(*constC.find( Value
::key( value ) ), value), NULL ); + // erase + ASSERT( my_c.unsafe_erase( Value
::key( value ) ), NULL ); + ASSERT( my_c.count( Value
::key( value ) ) == 0, NULL ); + // insert + std::pair res = my_c.insert( value ); + ASSERT( Harness::IsEqual()(*res.first, value), NULL ); + ASSERT( res.second, NULL); + // erase + Iterator it = res.first; + it++; + ASSERT( my_c.unsafe_erase( res.first ) == it, NULL ); + // insert + ASSERT( Harness::IsEqual()(*my_c.insert( my_c.begin(), value ), value), NULL ); + // equal_range + std::pair r1 = my_c.equal_range( Value
::key( value ) ); + ASSERT( Harness::IsEqual()(*r1.first, value) && ++r1.first == r1.second, NULL ); + std::pair r2 = constC.equal_range( Value
::key( value ) ); + ASSERT( Harness::IsEqual()(*r2.first, value) && ++r2.first == r2.second, NULL ); + + TestMapSpecificMethods( my_c, value ); + } +}; + +#include "tbb/task_scheduler_init.h" + +template +void CommonExamine( Table c, const std::list lst) { + typedef typename Table::value_type ValueType; + + ASSERT( !c.empty() && c.size() == lst.size() && c.max_size() >= c.size(), NULL ); + + std::for_each( lst.begin(), lst.end(), CheckValue( c ) ); + + std::vector< tbb::atomic > marks( lst.size() ); + + TestRange( lst, marks ).doTestRange( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + TestRange( lst, marks ).doTestRange( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + const Table constC = c; + ASSERT( c.size() == constC.size(), NULL ); + + TestRange( lst, marks ).doTestRange( constC.cbegin(), constC.cend() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + tbb::task_scheduler_init init; + + tbb::parallel_for( c.range(), TestRange( lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + tbb::parallel_for( constC.range( ), TestRange( lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + Table c2; + typename std::list::const_iterator begin5 = lst.begin(); + std::advance( begin5, 5 ); + c2.insert( lst.begin(), begin5 ); + std::for_each( lst.begin(), begin5, CheckValue( c2 ) ); + + c2.swap( c ); + ASSERT( c2.size() == lst.size(), NULL ); + ASSERT( c.size() == 5, NULL ); + std::for_each( lst.begin(), lst.end(), CheckValue( c2 ) ); + + c2.clear(); + ASSERT( c2.size() == 0, NULL ); + + typename Table::allocator_type a = c.get_allocator(); + ValueType *ptr = a.allocate( 1 ); + ASSERT( ptr, NULL ); + a.deallocate( ptr, 1 ); +} + +// overload for set and multiset +// second argument is needed just for right deduction +template +void TestSetCommonTypes() { + Checker CheckTypes; + const int NUMBER = 10; + + std::list arrInt; + for ( int i = 0; i( arrInt ); + + std::list< tbb::atomic > arrTbb(NUMBER); + int seq = 0; + for ( std::list< tbb::atomic >::iterator it = arrTbb.begin(); it != arrTbb.end(); ++it, ++seq ) *it = seq; + CheckTypes.template check( arrTbb ); + +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN + std::list< std::reference_wrapper > arrRef; + for ( std::list::iterator it = arrInt.begin( ); it != arrInt.end( ); ++it ) + arrRef.push_back( std::reference_wrapper(*it) ); + CheckTypes.template check( arrRef ); +#endif /* __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN */ + +#if __TBB_CPP11_SMART_POINTERS_PRESENT + std::list< std::shared_ptr > arrShr; + for ( int i = 0; i( i ) ); + CheckTypes.template check( arrShr ); + + std::list< std::weak_ptr > arrWk; + std::copy( arrShr.begin( ), arrShr.end( ), std::back_inserter( arrWk ) ); + CheckTypes.template check( arrWk ); +#else + REPORT( "Known issue: C++11 smart pointer tests are skipped.\n" ); +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ +} + +template +void TestMapCommonTypes() { + Checker CheckTypes; + const int NUMBER = 10; + + std::list< std::pair > arrIntInt; + for ( int i = 0; i < NUMBER; ++i ) arrIntInt.push_back( std::make_pair( i, NUMBER - i ) ); + CheckTypes.template check( arrIntInt ); + + std::list< std::pair< const int, tbb::atomic > > arrIntTbb; + for ( int i = 0; i < NUMBER; ++i ) { + tbb::atomic b; + b = NUMBER - i; + arrIntTbb.push_back( std::make_pair( i, b ) ); + } + CheckTypes.template check( arrIntTbb ); + +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN + std::list< std::pair, int> > arrRefInt; + for ( std::list< std::pair >::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) + arrRefInt.push_back( std::make_pair( std::reference_wrapper( it->first ), it->second ) ); + CheckTypes.template check( arrRefInt ); + + std::list< std::pair > > arrIntRef; + for ( std::list< std::pair >::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) { + // Using std::make_pair below causes compilation issues with early implementations of std::reference_wrapper. + arrIntRef.push_back( std::pair >( it->first, std::reference_wrapper( it->second ) ) ); + } + CheckTypes.template check( arrIntRef ); +#endif /* __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN */ + +#if __TBB_CPP11_SMART_POINTERS_PRESENT + std::list< std::pair< const std::shared_ptr, std::shared_ptr > > arrShrShr; + for ( int i = 0; i < NUMBER; ++i ) { + const int NUMBER_minus_i = NUMBER - i; + arrShrShr.push_back( std::make_pair( std::make_shared( i ), std::make_shared( NUMBER_minus_i ) ) ); + } + CheckTypes.template check( arrShrShr ); + + std::list< std::pair< const std::weak_ptr, std::weak_ptr > > arrWkWk; + std::copy( arrShrShr.begin(), arrShrShr.end(), std::back_inserter( arrWkWk ) ); + CheckTypes.template check( arrWkWk ); + +#else + REPORT( "Known issue: C++11 smart pointer tests are skipped.\n" ); +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ +} + + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT || __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +namespace node_handling{ + template + bool compare_handle_getters( + const Handle& node, const std::pair& expected + ) { + return node.key() == expected.first && node.mapped() == expected.second; + } + + template + bool compare_handle_getters( const Handle& node, const typename Handle::value_type& value) { + return node.value() == value; + } + + template + void set_node_handle_value( + Handle& node, const std::pair& value + ) { + node.key() = value.first; + node.mapped() = value.second; + } + + template + void set_node_handle_value( Handle& node, const typename Handle::value_type& value) { + node.value() = value; + } + + template + void TestTraits() { + ASSERT( !std::is_copy_constructible::value, + "Node handle: Handle is copy constructable" ); + ASSERT( !std::is_copy_assignable::value, + "Node handle: Handle is copy assignable" ); + ASSERT( std::is_move_constructible::value, + "Node handle: Handle is not move constructable" ); + ASSERT( std::is_move_assignable::value, + "Node handle: Handle is not move constructable" ); + ASSERT( std::is_default_constructible::value, + "Node handle: Handle is not default constructable" ); + ASSERT( std::is_destructible::value, + "Node handle: Handle is not destructible" ); + } + + template + void TestHandle( Table test_table ) { + ASSERT( test_table.size()>1, "Node handle: Container must contains 2 or more elements" ); + // Initialization + using node_type = typename Table::node_type; + + TestTraits(); + + // Default Ctor and empty function + node_type nh; + ASSERT( nh.empty(), "Node handle: Node is not empty after initialization" ); + + // Move Assign + // key/mapped/value function + auto expected_value = *test_table.begin(); + + nh = test_table.unsafe_extract(test_table.begin()); + ASSERT( !nh.empty(), "Node handle: Node handle is empty after valid move assigning" ); + ASSERT( compare_handle_getters(nh,expected_value), + "Node handle: After valid move assigning " + "node handle does not contains expected value"); + + // Move Ctor + // key/mapped/value function + node_type nh2(std::move(nh)); + ASSERT( nh.empty(), "Node handle: After valid move construction node handle is empty" ); + ASSERT( !nh2.empty(), "Node handle: After valid move construction " + "argument hode handle was not moved" ); + ASSERT( compare_handle_getters(nh2,expected_value), + "Node handle: After valid move construction " + "node handle does not contains expected value" ); + + // Bool conversion + ASSERT( nh2, "Node handle: Wrong not handle bool conversion" ); + + // Change key/mapped/value of node handle + auto expected_value2 = *test_table.begin(); + set_node_handle_value(nh2, expected_value2); + ASSERT( compare_handle_getters(nh2, expected_value2), + "Node handle: Wrong node handle key/mapped/value changing behavior" ); + + // Member/non member swap check + node_type empty_node; + // We extract this element for nh2 and nh3 difference + test_table.unsafe_extract(test_table.begin()); + auto expected_value3 = *test_table.begin(); + node_type nh3(test_table.unsafe_extract(test_table.begin())); + + // Both of node handles are not empty + nh3.swap(nh2); + ASSERT( compare_handle_getters(nh3, expected_value2), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( compare_handle_getters(nh2, expected_value3), + "Node handle: Wrong node handle swap behavior" ); + + std::swap(nh2,nh3); + ASSERT( compare_handle_getters(nh3, expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( compare_handle_getters(nh2, expected_value2), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( !nh2.empty(), "Node handle: Wrong node handle swap behavior" ); + ASSERT( !nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + + // One of nodes is empty + nh3.swap(empty_node); + ASSERT( compare_handle_getters(std::move(empty_node), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + + std::swap(empty_node, nh3); + ASSERT( compare_handle_getters(std::move(nh3), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( empty_node.empty(), "Node handle: Wrong node handle swap behavior" ); + + empty_node.swap(nh3); + ASSERT( compare_handle_getters(std::move(empty_node), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + } + + template + typename Table::node_type GenerateNodeHandle(const typename Table::value_type& value) { + Table temp_table; + temp_table.insert(value); + return temp_table.unsafe_extract(temp_table.cbegin()); + } + + template + void IteratorAssertion( const Table& table, + const typename Table::iterator& result, + const typename Table::value_type* node_value = nullptr ) { + if (node_value==nullptr) { + ASSERT( result==table.end(), "Insert: Result iterator does not " + "contains end pointer after empty node insertion" ); + } else { + if (!Table::allow_multimapping) { + ASSERT( result==table.find(Value
::key( *node_value )) && + result != table.end(), + "Insert: After node insertion result iterator" + " doesn't contains address to equal element in table" ); + } else { + ASSERT( *result==*node_value, "Insert: Result iterator contains" + "wrong content after successful insertion" ); + + for (auto it = table.begin(); it != table.end(); ++it) { + if (it == result) return; + } + ASSERT( false, "Insert: After successful insertion result " + "iterator contains address that is not in the table" ); + } + } + } + // overload for multitable or insertion with hint iterator + template + void InsertAssertion( const Table& table, + const typename Table::iterator& result, + bool, + const typename Table::value_type* node_value = nullptr ) { + IteratorAssertion(table, result, node_value); + } + + // Not multitable overload + template + void InsertAssertion( const Table& table, + const std::pair& result, + bool second_value, + const typename Table::value_type* node_value = nullptr ) { + IteratorAssertion(table, result.first, node_value); + + ASSERT( result.second == second_value || Table::allow_multimapping, + "Insert: Returned bool wrong value after node insertion" ); + } + +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + // Internal func for testing + // Can't delete ref from "Table" argument because hint must point to element of table + namespace { + template + void TestInsertOverloads( Table& table_to_insert, + const typename Table::value_type &value, const Hint&... hint ) { + // Insert empty element + typename Table::node_type nh; + + auto table_size = table_to_insert.size(); + auto result = table_to_insert.insert(hint..., std::move(nh)); + InsertAssertion(table_to_insert, result, /*second_value*/ false); + ASSERT( table_to_insert.size() == table_size, + "Insert: After empty node insertion table size changed" ); + + // Standard insertion + nh = GenerateNodeHandle
(value); + + result = table_to_insert.insert(hint..., std::move(nh)); + ASSERT( nh.empty(), "Insert: Not empty handle after successful insertion" ); + InsertAssertion(table_to_insert, result, /*second_value*/ true, &value); + + // Insert existing node + nh = GenerateNodeHandle
(value); + + result = table_to_insert.insert(hint..., std::move(nh)); + + InsertAssertion(table_to_insert, result, /*second_value*/ false, &value); + + if (Table::allow_multimapping){ + ASSERT( nh.empty(), "Insert: Failed insertion to multitable" ); + } else { + ASSERT( !nh.empty() , "Insert: Empty handle after failed insertion" ); + ASSERT( compare_handle_getters( std::move(nh), value ), + "Insert: Existing data does not equal to the one being inserted" ); + } + } + } + + template + void TestInsert( Table table, const typename Table::value_type & value) { + ASSERT( !table.empty(), "Insert: Map should contains 1 or more elements" ); + Table table_backup(table); + TestInsertOverloads(table, value); + TestInsertOverloads(table_backup, value, table_backup.begin()); + } +#endif /*__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT*/ + + template + void TestExtract( Table table_for_extract, typename Table::key_type new_key ) { + ASSERT( table_for_extract.size()>1, "Extract: Container must contains 2 or more element" ); + ASSERT( table_for_extract.find(new_key)==table_for_extract.end(), + "Extract: Table must not contains new element!"); + + // Extract new element + auto nh = table_for_extract.unsafe_extract(new_key); + ASSERT( nh.empty(), "Extract: Node handle is not empty after wrong key extraction" ); + + // Valid key extraction + auto expected_value = *table_for_extract.cbegin(); + auto key = Value
::key( expected_value ); + auto count = table_for_extract.count(key); + + nh = table_for_extract.unsafe_extract(key); + ASSERT( !nh.empty(), + "Extract: After successful extraction by key node handle is empty" ); + ASSERT( compare_handle_getters(std::move(nh), expected_value), + "Extract: After successful extraction by key node handle contains wrong value" ); + ASSERT( table_for_extract.count(key) == count - 1, + "Extract: After successful node extraction by key, table still contains this key" ); + + // Valid iterator overload + auto expected_value2 = *table_for_extract.cbegin(); + auto key2 = Value
::key( expected_value2 ); + auto count2 = table_for_extract.count(key2); + + nh = table_for_extract.unsafe_extract(table_for_extract.cbegin()); + ASSERT( !nh.empty(), + "Extract: After successful extraction by iterator node handle is empty" ); + ASSERT( compare_handle_getters(std::move(nh), expected_value2), + "Extract: After successful extraction by iterator node handle contains wrong value" ); + ASSERT( table_for_extract.count(key2) == count2 - 1, + "Extract: After successful extraction table also contains this element" ); + } + + // All test exclude merge + template + void NodeHandlingTests ( const Table& table, + const typename Table::value_type& new_value) { + TestHandle(table); +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + TestInsert(table, new_value); +#endif /*__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT*/ + TestExtract(table, Value
::key( new_value )); + } + + template + void TestMerge( TableType1 table1, TableType2&& table2 ) { + using Table2PureType = typename std::decay::type; + // Initialization + TableType1 table1_backup = table1; + // For copying lvalue + Table2PureType table2_backup = table2; + + table1.merge(std::forward(table2)); + for (auto it: table2) { + ASSERT( table1.find( Value::key( it ) ) != table1.end(), + "Merge: Some key(s) was not merged" ); + } + + // After the following step table1 will contains only merged elements from table2 + for (auto it: table1_backup) { + table1.unsafe_extract(Value::key( it )); + } + // After the following step table2_backup will contains only merged elements from table2 + for (auto it: table2) { + table2_backup.unsafe_extract(Value::key( it )); + } + + ASSERT ( table1.size() == table2_backup.size(), "Merge: Size of tables is not equal" ); + for (auto it: table2_backup) { + ASSERT( table1.find( Value::key( it ) ) != table1.end(), + "Merge: Wrong merge behavior" ); + } + } + + // Testing of rvalue and lvalue overloads + template + void TestMergeOverloads( const TableType1& table1, TableType2 table2 ) { + TableType2 table_backup(table2); + TestMerge(table1, table2); + TestMerge(table1, std::move(table_backup)); + } + + template + void TestMergeTransposition( Table table1, Table table2, + MultiTable multitable1, MultiTable multitable2 ) { + Table empty_map; + MultiTable empty_multimap; + + // Map transpositions + node_handling::TestMergeOverloads(table1, table2); + node_handling::TestMergeOverloads(table1, empty_map); + node_handling::TestMergeOverloads(empty_map, table2); + + // Multimap transpositions + node_handling::TestMergeOverloads(multitable1, multitable2); + node_handling::TestMergeOverloads(multitable1, empty_multimap); + node_handling::TestMergeOverloads(empty_multimap, multitable2); + + // Map/Multimap transposition + node_handling::TestMergeOverloads(table1, multitable1); + node_handling::TestMergeOverloads(multitable2, table2); + } + + template + void AssertionConcurrentMerge ( Table start_data, Table src_table, std::vector
tables, + std::true_type) { + ASSERT( src_table.size() == start_data.size()*tables.size(), + "Merge: Incorrect merge for some elements" ); + + for(auto it: start_data) { + ASSERT( src_table.count( Value
::key( it ) ) == + start_data.count( Value
::key( it ) )*tables.size(), + "Merge: Incorrect merge for some element" ); + } + + for (size_t i = 0; i < tables.size(); i++) { + ASSERT( tables[i].empty(), "Merge: Some elements was not merged" ); + } + } + + template + void AssertionConcurrentMerge ( Table start_data, Table src_table, std::vector
tables, + std::false_type) { + Table expected_result; + for (auto table: tables) + for (auto it: start_data) { + // If we cannot find some element in some table, then it has been moved + if (table.find( Value
::key( it ) ) == table.end()){ + bool result = expected_result.insert( it ).second; + ASSERT( result, "Merge: Some element was merged twice or was not " + "returned to his owner after unsuccessful merge"); + } + } + + ASSERT( expected_result.size() == src_table.size() && start_data.size() == src_table.size(), + "Merge: wrong size of result table"); + for (auto it: expected_result) { + if ( src_table.find( Value
::key( it ) ) != src_table.end() && + start_data.find( Value
::key( it ) ) != start_data.end() ){ + src_table.unsafe_extract(Value
::key( it )); + start_data.unsafe_extract(Value
::key( it )); + } else { + ASSERT( false, "Merge: Incorrect merge for some element" ); + } + } + + ASSERT( src_table.empty()&&start_data.empty(), "Merge: Some elements were not merged" ); + } + + template + void TestConcurrentMerge (const Table& table_data) { + for (auto num_threads = MinThread + 1; num_threads <= MaxThread; num_threads++){ + std::vector
tables; + Table src_table; + + for (auto j = 0; j < num_threads; j++){ + tables.push_back(table_data); + } + + NativeParallelFor( num_threads, [&](size_t index){ src_table.merge(tables[index]); } ); + + AssertionConcurrentMerge( table_data, src_table, tables, + std::integral_constant{}); + } + } + + + template + void TestNodeHandling(){ + Table table; + + for (int i = 1; i < 5; i++) + table.insert(Value
::make(i)); + + if (Table::allow_multimapping) + table.insert(Value
::make(4)); + + node_handling::NodeHandlingTests(table, Value
::make(5)); + } + + template + void TestMerge(int size){ + TableType1 table1_1; + TableType1 table1_2; + int i = 1; + for (; i < 5; ++i) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + if (TableType1::allow_multimapping) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + + TableType2 table2_1; + TableType2 table2_2; + for (i = 3; i < 7; ++i) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + if (TableType2::allow_multimapping) { + table2_1.insert(Value::make(i)); + table2_2.insert(Value::make(i*i)); + } + + node_handling::TestMergeTransposition(table1_1, table1_2, + table2_1, table2_2); + + TableType1 table1_3; + for (i = 0; i::make(i)); + } + node_handling::TestConcurrentMerge(table1_3); + + TableType2 table2_3; + for (i = 0; i::make(i)); + } + node_handling::TestConcurrentMerge(table2_3); +} +} +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT || __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT diff --git a/src/tbb/src/test/test_concurrent_hash_map.cpp b/src/tbb/src/test/test_concurrent_hash_map.cpp new file mode 100644 index 00000000..1bc4de27 --- /dev/null +++ b/src/tbb/src/test/test_concurrent_hash_map.cpp @@ -0,0 +1,1677 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef TBB_USE_PERFORMANCE_WARNINGS +#define TBB_USE_PERFORMANCE_WARNINGS 1 +#endif + +// Our tests usually include the header under test first. But this test needs +// to use the preprocessor to edit the identifier runtime_warning in concurrent_hash_map.h. +// Hence we include a few other headers before doing the abusive edit. +#include "tbb/tbb_stddef.h" /* Defines runtime_warning */ +#include "harness_assert.h" /* Prerequisite for defining hooked_warning */ + +// The symbol internal::runtime_warning is normally an entry point into the TBB library. +// Here for sake of testing, we define it to be hooked_warning, a routine peculiar to this unit test. +#define runtime_warning hooked_warning + +static bool bad_hashing = false; + +namespace tbb { + namespace internal { + static void hooked_warning( const char* /*format*/, ... ) { + ASSERT(bad_hashing, "unexpected runtime_warning: bad hashing"); + } + } // namespace internal +} // namespace tbb +#define __TBB_EXTRA_DEBUG 1 // enables additional checks +#include "tbb/concurrent_hash_map.h" + +// Restore runtime_warning as an entry point into the TBB library. +#undef runtime_warning + +namespace Jungle { + struct Tiger {}; + size_t tbb_hasher( const Tiger& ) {return 0;} +} + +#if !defined(_MSC_VER) || _MSC_VER>=1400 || __INTEL_COMPILER +void test_ADL() { + tbb::tbb_hash_compare::hash(Jungle::Tiger()); // Instantiation chain finds tbb_hasher via Argument Dependent Lookup +} +#endif + +struct UserDefinedKeyType { +}; + +namespace tbb { + // Test whether tbb_hash_compare can be partially specialized as stated in Reference manual. + template<> struct tbb_hash_compare { + size_t hash( UserDefinedKeyType ) const {return 0;} + bool equal( UserDefinedKeyType /*x*/, UserDefinedKeyType /*y*/ ) {return true;} + }; +} + +#include "harness_runtime_loader.h" + +tbb::concurrent_hash_map TestInstantiationWithUserDefinedKeyType; + +// Test whether a sufficient set of headers were included to instantiate a concurrent_hash_map. OSS Bug #120 (& #130): +// http://www.threadingbuildingblocks.org/bug_desc.php?id=120 +tbb::concurrent_hash_map,const char*>,int> TestInstantiation; + +#include "tbb/parallel_for.h" +#include "tbb/blocked_range.h" +#include "tbb/atomic.h" +#include "tbb/tick_count.h" +#include "harness.h" +#include "harness_allocator.h" + +class MyException : public std::bad_alloc { +public: + virtual const char *what() const throw() __TBB_override { return "out of items limit"; } + virtual ~MyException() throw() {} +}; + +/** Has tightly controlled interface so that we can verify + that concurrent_hash_map uses only the required interface. */ +class MyKey { +private: + void operator=( const MyKey& ); // Deny access + int key; + friend class MyHashCompare; + friend class YourHashCompare; +public: + static MyKey make( int i ) { + MyKey result; + result.key = i; + return result; + } + int value_of() const {return key;} +}; +//TODO: unify with Harness::Foo ? +tbb::atomic MyDataCount; +long MyDataCountLimit = 0; + +class MyData { +protected: + friend class MyData2; + int data; + enum state_t { + LIVE=0x1234, + DEAD=0x5678 + } my_state; + void operator=( const MyData& ); // Deny access +public: + MyData(int i = 0) { + my_state = LIVE; + data = i; + if(MyDataCountLimit && MyDataCount + 1 >= MyDataCountLimit) + __TBB_THROW( MyException() ); + ++MyDataCount; + } + MyData( const MyData& other ) { + ASSERT( other.my_state==LIVE, NULL ); + my_state = LIVE; + data = other.data; + if(MyDataCountLimit && MyDataCount + 1 >= MyDataCountLimit) + __TBB_THROW( MyException() ); + ++MyDataCount; + } + ~MyData() { + --MyDataCount; + my_state = DEAD; + } + static MyData make( int i ) { + MyData result; + result.data = i; + return result; + } + int value_of() const { + ASSERT( my_state==LIVE, NULL ); + return data; + } + void set_value( int i ) { + ASSERT( my_state==LIVE, NULL ); + data = i; + } + bool operator==( const MyData& other ) const { + ASSERT( other.my_state==LIVE, NULL ); + ASSERT( my_state==LIVE, NULL ); + return data == other.data; + } +}; + +class MyData2 : public MyData { +public: + MyData2( ) {} + MyData2( const MyData& other ) { + ASSERT( other.my_state==LIVE, NULL ); + ASSERT( my_state==LIVE, NULL ); + data = other.data; + } + void operator=( const MyData& other ) { + ASSERT( other.my_state==LIVE, NULL ); + ASSERT( my_state==LIVE, NULL ); + data = other.data; + } + void operator=( const MyData2& other ) { + ASSERT( other.my_state==LIVE, NULL ); + ASSERT( my_state==LIVE, NULL ); + data = other.data; + } + bool operator==( const MyData2& other ) const { + ASSERT( other.my_state==LIVE, NULL ); + ASSERT( my_state==LIVE, NULL ); + return data == other.data; + } +}; + +class MyHashCompare { +public: + bool equal( const MyKey& j, const MyKey& k ) const { + return j.key==k.key; + } + unsigned long hash( const MyKey& k ) const { + return k.key; + } +}; + +class YourHashCompare { +public: + bool equal( const MyKey& j, const MyKey& k ) const { + return j.key==k.key; + } + unsigned long hash( const MyKey& ) const { + return 1; + } +}; + +typedef local_counting_allocator > MyAllocator; +typedef tbb::concurrent_hash_map MyTable; +typedef tbb::concurrent_hash_map MyTable2; +typedef tbb::concurrent_hash_map YourTable; + +template +inline void CheckAllocator(MyTable &table, size_t expected_allocs, size_t expected_frees, bool exact = true) { + size_t items_allocated = table.get_allocator().items_allocated, items_freed = table.get_allocator().items_freed; + size_t allocations = table.get_allocator().allocations, frees = table.get_allocator().frees; + REMARK("checking allocators: items %u/%u, allocs %u/%u\n", + unsigned(items_allocated), unsigned(items_freed), unsigned(allocations), unsigned(frees) ); + ASSERT( items_allocated == allocations, NULL); ASSERT( items_freed == frees, NULL); + if(exact) { + ASSERT( allocations == expected_allocs, NULL); ASSERT( frees == expected_frees, NULL); + } else { + ASSERT( allocations >= expected_allocs, NULL); ASSERT( frees >= expected_frees, NULL); + ASSERT( allocations - frees == expected_allocs - expected_frees, NULL ); + } +} + +inline bool UseKey( size_t i ) { + return (i&3)!=3; +} + +struct Insert { + static void apply( MyTable& table, int i ) { + if( UseKey(i) ) { + if( i&4 ) { + MyTable::accessor a; + table.insert( a, MyKey::make(i) ); + if( i&1 ) + (*a).second.set_value(i*i); + else + a->second.set_value(i*i); + } else + if( i&1 ) { + MyTable::accessor a; + table.insert( a, std::make_pair(MyKey::make(i), MyData(i*i)) ); + ASSERT( (*a).second.value_of()==i*i, NULL ); + } else { + MyTable::const_accessor ca; + table.insert( ca, std::make_pair(MyKey::make(i), MyData(i*i)) ); + ASSERT( ca->second.value_of()==i*i, NULL ); + } + } + } +}; + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include "test_container_move_support.h" +typedef tbb::concurrent_hash_map DataStateTrackedTable; + +struct RvalueInsert { + static void apply( DataStateTrackedTable& table, int i ) { + DataStateTrackedTable::accessor a; + ASSERT( (table.insert( a, std::make_pair(MyKey::make(i), Foo(i + 1)))),"already present while should not ?" ); + ASSERT( (*a).second == i + 1, NULL ); + ASSERT( (*a).second.state == Harness::StateTrackableBase::MoveInitialized, ""); + } +}; + +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +struct Emplace { + static void apply( DataStateTrackedTable& table, int i ) { + DataStateTrackedTable::accessor a; + ASSERT( (table.emplace( a, MyKey::make(i), (i + 1))),"already present while should not ?" ); + ASSERT( (*a).second == i + 1, NULL ); + ASSERT( (*a).second.state == Harness::StateTrackableBase::DirectInitialized, ""); + } +}; +#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +#if __TBB_INITIALIZER_LISTS_PRESENT +struct InsertInitList { + static void apply( MyTable& table, int i ) { + if ( UseKey( i ) ) { + // TODO: investigate why the following sequence causes an additional allocation sometimes: + // table.insert( MyTable::value_type( MyKey::make( i ), i*i ) ); + // table.insert( MyTable::value_type( MyKey::make( i ), i*i+1 ) ); + std::initializer_list il = { MyTable::value_type( MyKey::make( i ), i*i )/*, MyTable::value_type( MyKey::make( i ), i*i+1 ) */ }; + table.insert( il ); + } + } +}; +#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ + +struct Find { + static void apply( MyTable& table, int i ) { + MyTable::accessor a; + const MyTable::accessor& ca = a; + bool b = table.find( a, MyKey::make(i) ); + ASSERT( b==!a.empty(), NULL ); + if( b ) { + if( !UseKey(i) ) + REPORT("Line %d: unexpected key %d present\n",__LINE__,i); + AssertSameType( &*a, static_cast(0) ); + ASSERT( ca->second.value_of()==i*i, NULL ); + ASSERT( (*ca).second.value_of()==i*i, NULL ); + if( i&1 ) + ca->second.set_value( ~ca->second.value_of() ); + else + (*ca).second.set_value( ~ca->second.value_of() ); + } else { + if( UseKey(i) ) + REPORT("Line %d: key %d missing\n",__LINE__,i); + } + } +}; + +struct FindConst { + static void apply( const MyTable& table, int i ) { + MyTable::const_accessor a; + const MyTable::const_accessor& ca = a; + bool b = table.find( a, MyKey::make(i) ); + ASSERT( b==(table.count(MyKey::make(i))>0), NULL ); + ASSERT( b==!a.empty(), NULL ); + ASSERT( b==UseKey(i), NULL ); + if( b ) { + AssertSameType( &*ca, static_cast(0) ); + ASSERT( ca->second.value_of()==~(i*i), NULL ); + ASSERT( (*ca).second.value_of()==~(i*i), NULL ); + } + } +}; + +tbb::atomic EraseCount; + +struct Erase { + static void apply( MyTable& table, int i ) { + bool b; + if(i&4) { + if(i&8) { + MyTable::const_accessor a; + b = table.find( a, MyKey::make(i) ) && table.erase( a ); + } else { + MyTable::accessor a; + b = table.find( a, MyKey::make(i) ) && table.erase( a ); + } + } else + b = table.erase( MyKey::make(i) ); + if( b ) ++EraseCount; + ASSERT( table.count(MyKey::make(i)) == 0, NULL ); + } +}; + +static const int IE_SIZE = 2; +tbb::atomic InsertEraseCount[IE_SIZE]; + +struct InsertErase { + static void apply( YourTable& table, int i ) { + if ( i%3 ) { + int key = i%IE_SIZE; + if ( table.insert( std::make_pair(MyKey::make(key), MyData2()) ) ) + ++InsertEraseCount[key]; + } else { + int key = i%IE_SIZE; + if( i&1 ) { + YourTable::accessor res; + if(table.find( res, MyKey::make(key) ) && table.erase( res ) ) + --InsertEraseCount[key]; + } else { + YourTable::const_accessor res; + if(table.find( res, MyKey::make(key) ) && table.erase( res ) ) + --InsertEraseCount[key]; + } + } + } +}; + +// Test for the deadlock discussed at: +// http://softwarecommunity.intel.com/isn/Community/en-US/forums/permalink/30253302/30253302/ShowThread.aspx#30253302 +struct InnerInsert { + static void apply( YourTable& table, int i ) { + YourTable::accessor a1, a2; + if(i&1) __TBB_Yield(); + table.insert( a1, MyKey::make(1) ); + __TBB_Yield(); + table.insert( a2, MyKey::make(1 + (1<<30)) ); // the same chain + table.erase( a2 ); // if erase by key it would lead to deadlock for single thread + } +}; + +#include "harness_barrier.h" +// Test for the misuse of constness +struct FakeExclusive : NoAssign { + Harness::SpinBarrier& barrier; + YourTable& table; + FakeExclusive(Harness::SpinBarrier& b, YourTable&t) : barrier(b), table(t) {} + void operator()( int i ) const { + if(i) { + YourTable::const_accessor real_ca; + // const accessor on non-const table acquired as reader (shared) + ASSERT( table.find(real_ca,MyKey::make(1)), NULL ); + barrier.wait(); // item can be erased + Harness::Sleep(10); // let it enter the erase + real_ca->second.value_of(); // check the state while holding accessor + } else { + YourTable::accessor fake_ca; + const YourTable &const_table = table; + // non-const accessor on const table acquired as reader (shared) + ASSERT( const_table.find(fake_ca,MyKey::make(1)), NULL ); + barrier.wait(); // readers acquired + // can mistakenly remove the item while other readers still refers to it + table.erase( fake_ca ); + } + } +}; + +template +class TableOperation: NoAssign { + MyTable& my_table; +public: + void operator()( const tbb::blocked_range& range ) const { + for( int i=range.begin(); i!=range.end(); ++i ) + Op::apply(my_table,i); + } + TableOperation( MyTable& table ) : my_table(table) {} +}; + +template +void DoConcurrentOperations( TableType& table, int n, const char* what, int nthread ) { + REMARK("testing %s with %d threads\n",what,nthread); + tbb::tick_count t0 = tbb::tick_count::now(); + tbb::parallel_for( tbb::blocked_range(0,n,100), TableOperation(table) ); + tbb::tick_count t1 = tbb::tick_count::now(); + REMARK("time for %s = %g with %d threads\n",what,(t1-t0).seconds(),nthread); +} + +//! Test traversing the table with an iterator. +void TraverseTable( MyTable& table, size_t n, size_t expected_size ) { + REMARK("testing traversal\n"); + size_t actual_size = table.size(); + ASSERT( actual_size==expected_size, NULL ); + size_t count = 0; + bool* array = new bool[n]; + memset( array, 0, n*sizeof(bool) ); + const MyTable& const_table = table; + MyTable::const_iterator ci = const_table.begin(); + for( MyTable::iterator i = table.begin(); i!=table.end(); ++i ) { + // Check iterator + int k = i->first.value_of(); + ASSERT( UseKey(k), NULL ); + ASSERT( (*i).first.value_of()==k, NULL ); + ASSERT( 0<=k && size_t(k) er = table.equal_range(i->first); + std::pair cer = const_table.equal_range(i->first); + ASSERT(cer.first == er.first && cer.second == er.second, NULL); + ASSERT(cer.first == i, NULL); + ASSERT(std::distance(cer.first, cer.second) == 1, NULL); + + // Check const_iterator + MyTable::const_iterator cic = ci++; + ASSERT( cic->first.value_of()==k, NULL ); + ASSERT( (*cic).first.value_of()==k, NULL ); + } + ASSERT( ci==const_table.end(), NULL ); + delete[] array; + if( count!=expected_size ) { + REPORT("Line %d: count=%ld but should be %ld\n",__LINE__,long(count),long(expected_size)); + } +} + +typedef tbb::atomic AtomicByte; + +template +struct ParallelTraverseBody: NoAssign { + const size_t n; + AtomicByte* const array; + ParallelTraverseBody( AtomicByte array_[], size_t n_ ) : + n(n_), + array(array_) + {} + void operator()( const RangeType& range ) const { + for( typename RangeType::iterator i = range.begin(); i!=range.end(); ++i ) { + int k = i->first.value_of(); + ASSERT( 0<=k && size_t(k)(array), 0, n*sizeof(AtomicByte) ); + MyTable::range_type r = table.range(10); + tbb::parallel_for( r, ParallelTraverseBody( array, n )); + Check( array, n, expected_size ); + + const MyTable& const_table = table; + memset( static_cast(array), 0, n*sizeof(AtomicByte) ); + MyTable::const_range_type cr = const_table.range(10); + tbb::parallel_for( cr, ParallelTraverseBody( array, n )); + Check( array, n, expected_size ); + + delete[] array; +} + +void TestInsertFindErase( int nthread ) { + int n=250000; + + // compute m = number of unique keys + int m = 0; + for( int i=0; i( table, n, "insert(std::initializer_list)", nthread ); + else +#endif + DoConcurrentOperations( table, n, "insert", nthread ); + ASSERT( MyDataCount == m, NULL ); + TraverseTable( table, n, m ); + ParallelTraverseTable( table, n, m ); + expected_allocs += m; + CheckAllocator( table, expected_allocs, expected_frees ); + + DoConcurrentOperations( table, n, "find", nthread ); + ASSERT( MyDataCount == m, NULL ); + CheckAllocator( table, expected_allocs, expected_frees ); + + DoConcurrentOperations( table, n, "find(const)", nthread ); + ASSERT( MyDataCount == m, NULL ); + CheckAllocator( table, expected_allocs, expected_frees ); + + EraseCount = 0; + DoConcurrentOperations( table, n, "erase", nthread ); + ASSERT( EraseCount == m, NULL ); + ASSERT( MyDataCount == 0, NULL ); + TraverseTable( table, n, 0 ); + expected_frees += m; + CheckAllocator( table, expected_allocs, expected_frees ); + + bad_hashing = true; + table.clear(); + bad_hashing = false; +#if __TBB_INITIALIZER_LISTS_PRESENT + } +#endif + + if(nthread > 1) { + YourTable ie_table; + for( int i=0; i(ie_table,n/2,"insert_erase",nthread); + for( int i=0; i(ie_table,2000,"inner insert",nthread); + Harness::SpinBarrier barrier(nthread); + REMARK("testing erase on fake exclusive accessor\n"); + NativeParallelFor( nthread, FakeExclusive(barrier, ie_table)); + } +} + +volatile int Counter; + +class AddToTable: NoAssign { + MyTable& my_table; + const int my_nthread; + const int my_m; +public: + AddToTable( MyTable& table, int nthread, int m ) : my_table(table), my_nthread(nthread), my_m(m) {} + void operator()( int ) const { + for( int i=0; isecond.set_value( 1 ); + else + a->second.set_value( a->second.value_of()+1 ); + k = a->second.value_of(); + } + if( k==my_nthread ) + Counter=i+1; + } + } +}; + +class RemoveFromTable: NoAssign { + MyTable& my_table; + const int my_m; +public: + RemoveFromTable( MyTable& table, int m ) : my_table(table), my_m(m) {} + void operator()(int) const { + for( int i=0; i(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast*>(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); +} + +template +void TestIteratorTraits() { + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + AssertSameType( static_cast(0), static_cast(0) ); + T x; + typename Iterator::reference xr = x; + typename Iterator::pointer xp = &x; + ASSERT( &xr==xp, NULL ); +} + +template +void TestIteratorAssignment( Iterator2 j ) { + Iterator1 i(j), k; + ASSERT( i==j, NULL ); ASSERT( !(i!=j), NULL ); + k = j; + ASSERT( k==j, NULL ); ASSERT( !(k!=j), NULL ); +} + +template +void TestRangeAssignment( Range2 r2 ) { + Range1 r1(r2); r1 = r2; +} +//------------------------------------------------------------------------ +// Test for copy constructor and assignment +//------------------------------------------------------------------------ + +template +static void FillTable( MyTable& x, int n ) { + for( int i=1; i<=n; ++i ) { + MyKey key( MyKey::make(-i) ); // hash values must not be specified in direct order + typename MyTable::accessor a; + bool b = x.insert(a,key); + ASSERT(b, NULL); + a->second.set_value( i*i ); + } +} + +template +static void CheckTable( const MyTable& x, int n ) { + ASSERT( x.size()==size_t(n), "table is different size than expected" ); + ASSERT( x.empty()==(n==0), NULL ); + ASSERT( x.size()<=x.max_size(), NULL ); + for( int i=1; i<=n; ++i ) { + MyKey key( MyKey::make(-i) ); + typename MyTable::const_accessor a; + bool b = x.find(a,key); + ASSERT( b, NULL ); + ASSERT( a->second.value_of()==i*i, NULL ); + } + int count = 0; + int key_sum = 0; + for( typename MyTable::const_iterator i(x.begin()); i!=x.end(); ++i ) { + ++count; + key_sum += -i->first.value_of(); + } + ASSERT( count==n, NULL ); + ASSERT( key_sum==n*(n+1)/2, NULL ); +} + +static void TestCopy() { + REMARK("testing copy\n"); + MyTable t1; + for( int i=0; i<10000; i=(i<100 ? i+1 : i*3) ) { + MyDataCount = 0; + + FillTable(t1,i); + // Do not call CheckTable(t1,i) before copying, it enforces rehashing + + MyTable t2(t1); + // Check that copy constructor did not mangle source table. + CheckTable(t1,i); + swap(t1, t2); + CheckTable(t1,i); + ASSERT( !(t1 != t2), NULL ); + + // Clear original table + t2.clear(); + swap(t2, t1); + CheckTable(t1,0); + + // Verify that copy of t1 is correct, even after t1 is cleared. + CheckTable(t2,i); + t2.clear(); + t1.swap( t2 ); + CheckTable(t1,0); + CheckTable(t2,0); + ASSERT( MyDataCount==0, "data leak?" ); + } +} + +void TestAssignment() { + REMARK("testing assignment\n"); + for( int i=0; i<1000; i=(i<30 ? i+1 : i*5) ) { + for( int j=0; j<1000; j=(j<30 ? j+1 : j*7) ) { + MyTable t1; + MyTable t2; + FillTable(t1,i); + FillTable(t2,j); + ASSERT( (t1 == t2) == (i == j), NULL ); + CheckTable(t2,j); + + MyTable& tref = t2=t1; + ASSERT( &tref==&t2, NULL ); + ASSERT( t1 == t2, NULL ); + CheckTable(t1,i); + CheckTable(t2,i); + + t1.clear(); + CheckTable(t1,0); + CheckTable(t2,i); + ASSERT( MyDataCount==i, "data leak?" ); + + t2.clear(); + CheckTable(t1,0); + CheckTable(t2,0); + ASSERT( MyDataCount==0, "data leak?" ); + } + } +} + +void TestIteratorsAndRanges() { + REMARK("testing iterators compliance\n"); + TestIteratorTraits(); + TestIteratorTraits(); + + MyTable v; + MyTable const &u = v; + + TestIteratorAssignment( u.begin() ); + TestIteratorAssignment( v.begin() ); + TestIteratorAssignment( v.begin() ); + // doesn't compile as expected: TestIteratorAssignment( u.begin() ); + + // check for non-existing + ASSERT(v.equal_range(MyKey::make(-1)) == std::make_pair(v.end(), v.end()), NULL); + ASSERT(u.equal_range(MyKey::make(-1)) == std::make_pair(u.end(), u.end()), NULL); + + REMARK("testing ranges compliance\n"); + TestRangeAssignment( u.range() ); + TestRangeAssignment( v.range() ); + TestRangeAssignment( v.range() ); + // doesn't compile as expected: TestRangeAssignment( u.range() ); + + REMARK("testing construction and insertion from iterators range\n"); + FillTable( v, 1000 ); + MyTable2 t(v.begin(), v.end()); + v.rehash(); + CheckTable(t, 1000); + t.insert(v.begin(), v.end()); // do nothing + CheckTable(t, 1000); + t.clear(); + t.insert(v.begin(), v.end()); // restore + CheckTable(t, 1000); + + REMARK("testing comparison\n"); + typedef tbb::concurrent_hash_map YourTable1; + typedef tbb::concurrent_hash_map YourTable2; + YourTable1 t1; + FillTable( t1, 10 ); + CheckTable(t1, 10 ); + YourTable2 t2(t1.begin(), t1.end()); + MyKey key( MyKey::make(-5) ); MyData2 data; + ASSERT(t2.erase(key), NULL); + YourTable2::accessor a; + ASSERT(t2.insert(a, key), NULL); + data.set_value(0); a->second = data; + ASSERT( t1 != t2, NULL); + data.set_value(5*5); a->second = data; + ASSERT( t1 == t2, NULL); +} + +void TestRehash() { + REMARK("testing rehashing\n"); + MyTable w; + w.insert( std::make_pair(MyKey::make(-5), MyData()) ); + w.rehash(); // without this, assertion will fail + MyTable::iterator it = w.begin(); + int i = 0; // check for non-rehashed buckets + for( ; it != w.end(); i++ ) + w.count( (it++)->first ); + ASSERT( i == 1, NULL ); + for( i=0; i<1000; i=(i<29 ? i+1 : i*2) ) { + for( int j=max(256+i, i*2); j<10000; j*=3 ) { + MyTable v; + FillTable( v, i ); + ASSERT(int(v.size()) == i, NULL); + ASSERT(int(v.bucket_count()) <= j, NULL); + v.rehash( j ); + ASSERT(int(v.bucket_count()) >= j, NULL); + CheckTable( v, i ); + } + } +} + +template > +class only_node_counting_allocator : public local_counting_allocator { + typedef local_counting_allocator base_type; +public: + template + struct rebind { + typedef only_node_counting_allocator::other,count_t> other; + }; + + only_node_counting_allocator() : base_type() {} + only_node_counting_allocator(const only_node_counting_allocator& a) : base_type(a) {} + + template + only_node_counting_allocator(const only_node_counting_allocator& a) : base_type(a) {} + + typename base_type::pointer allocate(const typename base_type::size_type n) { + if ( n > 1) { + return base_alloc_t::allocate(n); + } else { + return base_type::allocate(n); + } + } +}; + +#if TBB_USE_EXCEPTIONS +void TestExceptions() { + typedef only_node_counting_allocator > allocator_t; + typedef tbb::concurrent_hash_map ThrowingTable; + enum methods { + zero_method = 0, + ctor_copy, op_assign, op_insert, + all_methods + }; + REMARK("testing exception-safety guarantees\n"); + ThrowingTable src; + FillTable( src, 1000 ); + ASSERT( MyDataCount==1000, NULL ); + + try { + for(int t = 0; t < 2; t++) // exception type + for(int m = zero_method+1; m < all_methods; m++) + { + allocator_t a; + if(t) MyDataCountLimit = 101; + else a.set_limits(101); + ThrowingTable victim(a); + MyDataCount = 0; + + try { + switch(m) { + case ctor_copy: { + ThrowingTable acopy(src, a); + } break; + case op_assign: { + victim = src; + } break; + case op_insert: { +#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT + // Insertion in cpp11 don't make copy constructions + // during the insertion, so we need to decrement limit + // to throw an exception in the right place and to prevent + // successful insertion of one unexpected item + if (MyDataCountLimit) + --MyDataCountLimit; +#endif + FillTable( victim, 1000 ); + } break; + default:; + } + ASSERT(false, "should throw an exception"); + } catch(std::bad_alloc &e) { + MyDataCountLimit = 0; + size_t size = victim.size(); + switch(m) { + case op_assign: + ASSERT( MyDataCount==100, "data leak?" ); + ASSERT( size>=100, NULL ); + CheckAllocator(victim, 100+t, t); + __TBB_fallthrough; + case ctor_copy: + CheckTable(src, 1000); + break; + case op_insert: + ASSERT( size==size_t(100-t), NULL ); + ASSERT( MyDataCount==100-t, "data leak?" ); + CheckTable(victim, 100-t); + CheckAllocator(victim, 100, t); + break; + + default:; // nothing to check here + } + REMARK("Exception %d: %s\t- ok ()\n", m, e.what()); + } + catch ( ... ) { + ASSERT ( __TBB_EXCEPTION_TYPE_INFO_BROKEN, "Unrecognized exception" ); + } + } + } catch(...) { + ASSERT(false, "unexpected exception"); + } + src.clear(); MyDataCount = 0; +} +#endif /* TBB_USE_EXCEPTIONS */ + + +#if __TBB_INITIALIZER_LISTS_PRESENT +#include "test_initializer_list.h" + +struct test_insert { + template + static void do_test( std::initializer_list il, container_type const& expected ) { + container_type vd; + vd.insert( il ); + ASSERT( vd == expected, "inserting with an initializer list failed" ); + } +}; + +void TestInitList(){ + using namespace initializer_list_support_tests; + REMARK("testing initializer_list methods \n"); + + typedef tbb::concurrent_hash_map ch_map_type; + std::initializer_list pairs_il = {{1,1},{2,2},{3,3},{4,4},{5,5}}; + + TestInitListSupportWithoutAssign( pairs_il ); + TestInitListSupportWithoutAssign( {} ); +} +#endif //if __TBB_INITIALIZER_LISTS_PRESENT + +#if __TBB_RANGE_BASED_FOR_PRESENT +#include "test_range_based_for.h" + +void TestRangeBasedFor(){ + using namespace range_based_for_support_tests; + + REMARK("testing range based for loop compatibility \n"); + typedef tbb::concurrent_hash_map ch_map; + ch_map a_ch_map; + + const int sequence_length = 100; + for (int i = 1; i <= sequence_length; ++i){ + a_ch_map.insert(ch_map::value_type(i,i)); + } + + ASSERT( range_based_for_accumulate(a_ch_map, pair_second_summer(), 0) == gauss_summ_of_int_sequence(sequence_length), "incorrect accumulated value generated via range based for ?"); +} +#endif //if __TBB_RANGE_BASED_FOR_PRESENT + +#include "harness_defs.h" + +// The helper to run a test only when a default construction is present. +template struct do_default_construction_test { + template void operator() ( FuncType func ) const { func(); } +}; +template <> struct do_default_construction_test { + template void operator()( FuncType ) const {} +}; + +template +class test_insert_by_key : NoAssign { + typedef typename Table::value_type value_type; + Table &my_c; + const value_type &my_value; +public: + test_insert_by_key( Table &c, const value_type &value ) : my_c(c), my_value(value) {} + void operator()() const { + { + typename Table::accessor a; + ASSERT( my_c.insert( a, my_value.first ), NULL ); + ASSERT( Harness::IsEqual()(a->first, my_value.first), NULL ); + a->second = my_value.second; + } { + typename Table::const_accessor ca; + ASSERT( !my_c.insert( ca, my_value.first ), NULL ); + ASSERT( Harness::IsEqual()(ca->first, my_value.first), NULL); + ASSERT( Harness::IsEqual()(ca->second, my_value.second), NULL); + } + } +}; + +#include +#include +#include +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT +#include +#endif + +template +class test_range : NoAssign { + typedef typename Table::value_type value_type; + Table &my_c; + const std::list &my_lst; + std::vector< tbb::atomic >& my_marks; +public: + test_range( Table &c, const std::list &lst, std::vector< tbb::atomic > &marks ) : my_c(c), my_lst(lst), my_marks(marks) { + std::fill( my_marks.begin(), my_marks.end(), false ); + } + void operator()( const Range &r ) const { do_test_range( r.begin(), r.end() ); } + void do_test_range( Iterator i, Iterator j ) const { + for ( Iterator it = i; it != j; ) { + Iterator it_prev = it++; + typename std::list::const_iterator it2 = std::search( my_lst.begin(), my_lst.end(), it_prev, it, Harness::IsEqual() ); + ASSERT( it2 != my_lst.end(), NULL ); + typename std::list::difference_type dist = std::distance( my_lst.begin(), it2 ); + ASSERT( !my_marks[dist], NULL ); + my_marks[dist] = true; + } + } +}; + +template +class check_value : NoAssign { + typedef typename Table::const_iterator const_iterator; + typedef typename Table::iterator iterator; + typedef typename Table::size_type size_type; + Table &my_c; +public: + check_value( Table &c ) : my_c(c) {} + void operator()(const typename Table::value_type &value ) { + const Table &const_c = my_c; + ASSERT( my_c.count( value.first ) == 1, NULL ); + { // tests with a const accessor. + typename Table::const_accessor ca; + // find + ASSERT( my_c.find( ca, value.first ), NULL); + ASSERT( !ca.empty() , NULL); + ASSERT( Harness::IsEqual()(ca->first, value.first), NULL ); + ASSERT( Harness::IsEqual()(ca->second, value.second), NULL ); + // erase + ASSERT( my_c.erase( ca ), NULL ); + ASSERT( my_c.count( value.first ) == 0, NULL ); + // insert (pair) + ASSERT( my_c.insert( ca, value ), NULL); + ASSERT( Harness::IsEqual()(ca->first, value.first), NULL ); + ASSERT( Harness::IsEqual()(ca->second, value.second), NULL ); + } { // tests with a non-const accessor. + typename Table::accessor a; + // find + ASSERT( my_c.find( a, value.first ), NULL); + ASSERT( !a.empty() , NULL); + ASSERT( Harness::IsEqual()(a->first, value.first), NULL ); + ASSERT( Harness::IsEqual()(a->second, value.second), NULL ); + // erase + ASSERT( my_c.erase( a ), NULL ); + ASSERT( my_c.count( value.first ) == 0, NULL ); + // insert + ASSERT( my_c.insert( a, value ), NULL); + ASSERT( Harness::IsEqual()(a->first, value.first), NULL ); + ASSERT( Harness::IsEqual()(a->second, value.second), NULL ); + } + // erase by key + ASSERT( my_c.erase( value.first ), NULL ); + ASSERT( my_c.count( value.first ) == 0, NULL ); + do_default_construction_test()(test_insert_by_key
( my_c, value )); + // insert by value + ASSERT( my_c.insert( value ) != default_construction_present, NULL ); + // equal_range + std::pair r1 = my_c.equal_range( value.first ); + iterator r1_first_prev = r1.first++; + ASSERT( Harness::IsEqual()( *r1_first_prev, value ) && Harness::IsEqual()( r1.first, r1.second ), NULL ); + std::pair r2 = const_c.equal_range( value.first ); + const_iterator r2_first_prev = r2.first++; + ASSERT( Harness::IsEqual()( *r2_first_prev, value ) && Harness::IsEqual()( r2.first, r2.second ), NULL ); + } +}; + +#include "tbb/task_scheduler_init.h" + +template +struct CompareTables { + template + static bool IsEqual( const T& t1, const T& t2 ) { + return (t1 == t2) && !(t1 != t2); + } +}; + +#if __TBB_CPP11_SMART_POINTERS_PRESENT +template +struct CompareTables< std::pair, std::weak_ptr > > { + template + static bool IsEqual( const T&, const T& ) { + /* do nothing for std::weak_ptr */ + return true; + } +}; +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ + +template +void Examine( Table c, const std::list &lst) { + typedef const Table const_table; + typedef typename Table::const_iterator const_iterator; + typedef typename Table::iterator iterator; + typedef typename Table::value_type value_type; + typedef typename Table::size_type size_type; + + ASSERT( !c.empty(), NULL ); + ASSERT( c.size() == lst.size(), NULL ); + ASSERT( c.max_size() >= c.size(), NULL ); + + const check_value cv(c); + std::for_each( lst.begin(), lst.end(), cv ); + + std::vector< tbb::atomic > marks( lst.size() ); + + test_range( c, lst, marks ).do_test_range( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + test_range( c, lst, marks ).do_test_range( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + tbb::task_scheduler_init init; + + typedef typename Table::range_type range_type; + tbb::parallel_for( c.range(), test_range( c, lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + const_table const_c = c; + ASSERT( CompareTables::IsEqual( c, const_c ), NULL ); + + typedef typename const_table::const_range_type const_range_type; + tbb::parallel_for( c.range(), test_range( const_c, lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + const size_type new_bucket_count = 2*c.bucket_count(); + c.rehash( new_bucket_count ); + ASSERT( c.bucket_count() >= new_bucket_count, NULL ); + + Table c2; + typename std::list::const_iterator begin5 = lst.begin(); + std::advance( begin5, 5 ); + c2.insert( lst.begin(), begin5 ); + std::for_each( lst.begin(), begin5, check_value( c2 ) ); + + c2.swap( c ); + ASSERT( CompareTables::IsEqual( c2, const_c ), NULL ); + ASSERT( c.size() == 5, NULL ); + std::for_each( lst.begin(), lst.end(), check_value(c2) ); + + tbb::swap( c, c2 ); + ASSERT( CompareTables::IsEqual( c, const_c ), NULL ); + ASSERT( c2.size() == 5, NULL ); + + c2.clear(); + ASSERT( CompareTables::IsEqual( c2, Table() ), NULL ); + + typename Table::allocator_type a = c.get_allocator(); + value_type *ptr = a.allocate(1); + ASSERT( ptr, NULL ); + a.deallocate( ptr, 1 ); +} + +template +struct debug_hash_compare : tbb::tbb_hash_compare {}; + +template +void TypeTester( const std::list &lst ) { + __TBB_ASSERT( lst.size() >= 5, "Array should have at least 5 elements" ); + typedef typename Value::first_type first_type; + typedef typename Value::second_type second_type; + typedef tbb::concurrent_hash_map ch_map; + debug_hash_compare compare; + // Construct an empty hash map. + ch_map c1; + c1.insert( lst.begin(), lst.end() ); + Examine( c1, lst ); +#if __TBB_INITIALIZER_LISTS_PRESENT && !__TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN + // Constructor from initializer_list. + typename std::list::const_iterator it = lst.begin(); + std::initializer_list il = { *it++, *it++, *it++ }; + ch_map c2( il ); + c2.insert( it, lst.end() ); + Examine( c2, lst ); + + // Constructor from initializer_list and compare object + ch_map c3( il, compare); + c3.insert( it, lst.end() ); + Examine( c3, lst ); + + // Constructor from initializer_list, compare object and allocator + ch_map c4( il, compare, typename ch_map::allocator_type()); + c4.insert( it, lst.end()); + Examine( c4, lst ); +#endif + // Copying constructor. + ch_map c5(c1); + Examine( c5, lst ); + // Construct with non-default allocator + typedef tbb::concurrent_hash_map< first_type,second_type,tbb::tbb_hash_compare,debug_allocator > ch_map_debug_alloc; + ch_map_debug_alloc c6; + c6.insert( lst.begin(), lst.end() ); + Examine( c6, lst ); + // Copying constructor + ch_map_debug_alloc c7(c6); + Examine( c7, lst ); + // Construction empty table with n preallocated buckets. + ch_map c8( lst.size() ); + c8.insert( lst.begin(), lst.end() ); + Examine( c8, lst ); + ch_map_debug_alloc c9( lst.size() ); + c9.insert( lst.begin(), lst.end() ); + Examine( c9, lst ); + // Construction with copying iteration range. + ch_map c10( c1.begin(), c1.end() ); + Examine( c10, lst ); + // Construction with copying iteration range and given allocator instance. + debug_allocator allocator; + ch_map_debug_alloc c11( lst.begin(), lst.end(), allocator ); + Examine( c11, lst ); + + typedef tbb::concurrent_hash_map< first_type,second_type,debug_hash_compare,typename ch_map::allocator_type> ch_map_debug_hash; + + // Constructor with two iterators and hash_compare + ch_map_debug_hash c12(c1.begin(), c1.end(), compare); + Examine( c12, lst ); + + ch_map_debug_hash c13(c1.begin(), c1.end(), compare, typename ch_map::allocator_type()); + Examine( c13, lst ); +} + +#if __TBB_CPP11_SMART_POINTERS_PRESENT +namespace tbb { + template<> struct tbb_hash_compare< const std::shared_ptr > { + static size_t hash( const std::shared_ptr& ptr ) { return static_cast( *ptr ) * interface5::internal::hash_multiplier; } + static bool equal( const std::shared_ptr& ptr1, const std::shared_ptr& ptr2 ) { return ptr1 == ptr2; } + }; + template<> struct tbb_hash_compare< const std::weak_ptr > { + static size_t hash( const std::weak_ptr& ptr ) { return static_cast( *ptr.lock() ) * interface5::internal::hash_multiplier; } + static bool equal( const std::weak_ptr& ptr1, const std::weak_ptr& ptr2 ) { return ptr1.lock() == ptr2.lock(); } + }; +} +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ + +void TestCPP11Types() { + const int NUMBER = 10; + + typedef std::pair int_int_t; + std::list arrIntInt; + for ( int i=0; i( arrIntInt ); + +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN + typedef std::pair, int> ref_int_t; + std::list arrRefInt; + for ( std::list::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) + arrRefInt.push_back( ref_int_t( it->first, it->second ) ); + TypeTester( arrRefInt ); + + typedef std::pair< const int, std::reference_wrapper > int_ref_t; + std::list arrIntRef; + for ( std::list::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) + arrIntRef.push_back( int_ref_t( it->first, it->second ) ); + TypeTester( arrIntRef ); +#else + REPORT("Known issue: C++11 reference wrapper tests are skipped.\n"); +#endif /* __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN*/ + + typedef std::pair< const int, tbb::atomic > int_tbb_t; + std::list arrIntTbb; + for ( int i=0; i b; + b = NUMBER-i; + arrIntTbb.push_back( int_tbb_t(i, b) ); + } + TypeTester( arrIntTbb ); + +#if __TBB_CPP11_SMART_POINTERS_PRESENT + typedef std::pair< const std::shared_ptr, std::shared_ptr > shr_shr_t; + std::list arrShrShr; + for ( int i=0; i(i), std::make_shared(NUMBER_minus_i) ) ); + } + TypeTester< /*default_construction_present = */true>( arrShrShr ); + + typedef std::pair< const std::weak_ptr, std::weak_ptr > wk_wk_t; + std::list< wk_wk_t > arrWkWk; + std::copy( arrShrShr.begin(), arrShrShr.end(), std::back_inserter(arrWkWk) ); + TypeTester< /*default_construction_present = */true>( arrWkWk ); +#else + REPORT("Known issue: C++11 smart pointer tests are skipped.\n"); +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ +} + +#if __TBB_CPP11_RVALUE_REF_PRESENT + +struct hash_map_move_traits : default_container_traits { + enum{ expected_number_of_items_to_allocate_for_steal_move = 0 }; + + template + struct hash_compare { + bool equal( const T& lhs, const T& rhs ) const { + return lhs==rhs; + } + size_t hash( const T& k ) const { + return tbb::tbb_hasher(k); + } + }; + template + struct apply { + typedef tbb::concurrent_hash_map, allocator_type > type; + }; + + typedef FooPairIterator init_iterator_type; + template + static bool equal(hash_map_type const& c, iterator begin, iterator end){ + bool equal_sizes = ( static_cast(std::distance(begin, end)) == c.size() ); + if (!equal_sizes) + return false; + + for (iterator it = begin; it != end; ++it ){ + if (c.count( (*it).first) == 0){ + return false; + } + } + return true; + } +}; + +void TestMoveSupport(){ + TestMoveConstructor(); + TestConstructorWithMoveIterators(); + TestMoveAssignOperator(); +#if TBB_USE_EXCEPTIONS + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorMemoryFailure(); + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorExceptionInElementCtor(); +#else + REPORT("Known issue: exception safety tests for C++11 move semantics support are skipped.\n"); +#endif //TBB_USE_EXCEPTIONS +} +#else +void TestMoveSupport(){ + REPORT("Known issue: tests for C++11 move semantics support are skipped.\n"); +} +#endif //__TBB_CPP11_RVALUE_REF_PRESENT + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template

+
include +
Include files required for compiling code that uses the library. +
examples +
Examples of how to use the library. +
python +
Python* API for Intel TBB. +