Extend documentation wrt mixing with other threading packages (#1702)

Co-authored-by: Alexandra <[email protected]>

Author: aleksei-fedotov

doc/GSG/next_steps.rst

@@ -19,9 +19,10 @@ After installing |short_name|, set the environment variables:
 
 .. tip::
 
-   oneTBB can coordinate with Intel(R) OpenMP on CPU resources usage 
-   to avoid excessive oversubscription when both runtimes are used within a process.
-   To enable this feature set up ``TCM_ENABLE`` environment variable to ``1``.
+   oneTBB can coordinate with Intel(R) OpenMP on CPU resources usage to avoid
+   excessive oversubscription when both runtimes are used within a process. To
+   enable this feature set ``TCM_ENABLE`` environment variable to ``1``. For
+   more details, see :ref:`avoid_cpu_overutilization`.
 
 
 Build and Run a Sample 

doc/main/examples_testing/CMakeLists.txt

@@ -39,3 +39,11 @@ file(GLOB_RECURSE DOC_EXAMPLES_LIST "${_reference_examples_path}/*.cpp" "${_user
 foreach(_doc_example_path IN LISTS DOC_EXAMPLES_LIST)
     add_doc_example(${_doc_example_path})
 endforeach()
+
+find_package(OpenMP)
+if (OpenMP_FOUND)
+    target_compile_options(tbb_mixing_other_runtimes_example PRIVATE "${OpenMP_CXX_FLAGS}")
+    target_link_options(tbb_mixing_other_runtimes_example PRIVATE "${OpenMP_CXX_FLAGS}")
+endif()
+
+

doc/main/tbb_userguide/appendix_B.rst

@@ -4,71 +4,107 @@ Appendix B Mixing With Other Threading Packages
 ===============================================
 
 
-|full_name| can be mixed with other
-threading packages. No special effort is required to use any part of
-oneTBB with other threading packages.
+Correct Interoperability
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use |short_name| with other threading packages. No additional
+effort is required.
 
 
 Here is an example that parallelizes an outer loop with OpenMP and an
-inner loop with oneTBB.
+inner loop with |short_name|.
 
+.. literalinclude:: ./examples/tbb_mixing_other_runtimes_example.cpp
+   :language: c++
+   :start-after: /*begin outer loop openmp with nested tbb*/
+   :end-before: /*end outer loop openmp with nested tbb*/
 
-::
+
+``#pragma omp parallel`` instructs OpenMP to create a team of
+threads. Each thread executes the code block statement associated with
+the directive.
+
+``#pragma omp for`` indicates that the compiler should distribute
+the iterations of the following loop among the threads in the existing
+thread team, enabling parallel execution of the loop body.
+
+
+See the similar example with the POSIX\* Threads:
+
+.. literalinclude:: ./examples/tbb_mixing_other_runtimes_example.cpp
+   :language: c++
+   :start-after: /*begin pthreads with tbb*/
+   :end-before: /*end pthreads with tbb*/
+
+
+.. _avoid_cpu_overutilization:
+
+Avoid CPU Overutilization
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While you can safely use |short_name| with other threading packages
+without affecting the execution correctness, running a large number of
+threads from multiple thread pools concurrently can lead to
+oversubscription. This may significantly overutilize system resources,
+affecting the execution performance.
 
 
-   int M, N;
-    
+Consider the previous example with nested parallelism, but with an
+OpenMP parallel region executed within the parallel loop:
 
-   struct InnerBody {
-       ...
-   };
-    
+.. literalinclude:: ./examples/tbb_mixing_other_runtimes_example.cpp
+   :language: c++
+   :start-after: /*begin outer loop tbb with nested omp*/
+   :end-before: /*end outer loop tbb with nested omp*/
 
-   void TBB_NestedInOpenMP() {
-   #pragma omp parallel
-       {
-   #pragma omp for
-           for( int i=0; i<M; ++ ) {
-               parallel_for( blocked_range<int>(0,N,10), InnerBody(i) );
-           }
-       }
-   }
 
+Due to the semantics of the OpenMP parallel region, this composition of
+parallel runtimes may result in a quadratic number of simultaneously
+running threads. Such oversubscription can degrade the performance.
 
-The details of ``InnerBody`` are omitted for brevity. The
-``#pragma omp parallel`` causes the OpenMP to create a team of threads,
-and each thread executes the block statement associated with the pragma.
-The ``#pragma omp for`` indicates that the compiler should use the
-previously created thread team to execute the loop in parallel.
 
+|short_name| solves this issue with Thread Composability Manager (TCM).
+It is an experimental CPU resource coordination layer that enables
+better cooperation between different threading runtimes.
 
-Here is the same example written using POSIX\* Threads.
 
+By default, TCM is disabled. To enable it, set ``TCM_ENABLE``
+environment variable to ``1``. To make sure it works as intended set
+``TCM_VERSION`` environment variable to ``1`` before running your
+application and check the output for lines starting with ``TCM:``. The
+``TCM: TCM_ENABLE 1`` line confirms that Thread Composability Manager is
+active.
+
+
+Example output:
 
 ::
 
+    TCM: VERSION            1.3.0
+    <...>
+    TCM: TCM_ENABLE         1
+
+
+When used with the OpenMP implementation of Intel(R) DPC++/C++ Compiler,
+TCM allows to avoid simultaneous scheduling of excessive threads in the
+scenarios similar to the one above.
+
+
+Submit feedback or ask questions about Thread Composability
+Manager through |short_name| `GitHub Issues
+<https://github.com/uxlfoundation/oneTBB/issues>`_ or `Discussions
+<https://github.com/uxlfoundation/oneTBB/discussions>`_.
+
+
+.. note::
+   Coordination on the use of CPU resources requires support for Thread
+   Composability Manager. For optimal coordination, make sure that each
+   threading package in your application integrates with TCM.
+
+
+.. rubric:: See also
 
-   int M, N;
-    
-
-   struct InnerBody {
-       ...
-   };
-    
-
-   void* OuterLoopIteration( void* args ) {
-       int i = (int)args;
-       parallel_for( blocked_range<int>(0,N,10), InnerBody(i) );
-   }
-    
-
-   void TBB_NestedInPThreads() {
-       std::vector<pthread_t> id( M );
-       // Create thread for each outer loop iteration
-       for( int i=0; i<M; ++i )
-           pthread_create( &id[i], NULL, OuterLoopIteration, NULL );
-       // Wait for outer loop threads to finish
-       for( int i=0; i<M; ++i )
-           pthread_join( &id[i], NULL );
-   } 
+* `End Parallel Runtime Scheduling Conflicts with Thread Composability
+  Manager
+  <https://www.intel.com/content/www/us/en/developer/videos/threading-composability-manager-with-onetbb.html>`_
 

doc/main/tbb_userguide/examples/tbb_mixing_other_runtimes_example.cpp

@@ -0,0 +1,123 @@
+#include <cstdint>
+#include <vector>
+#include <omp.h>
+#ifndef _WIN32
+#include <pthread.h>
+#endif
+#include <oneapi/tbb/global_control.h>
+#include <oneapi/tbb/parallel_for.h>
+
+namespace nesting_tbb {
+
+/*begin outer loop openmp with nested tbb*/
+int M, N;
+
+struct InnerBody {
+    int i;
+    void operator()(tbb::blocked_range<int> const& r) const {
+        for (auto j = r.begin(); j != r.end(); ++j) {
+            // do the work for (i, j) element
+        }
+    }
+};
+
+void TBB_NestedInOpenMP() {
+#pragma omp parallel
+    {
+#pragma omp for
+        for(int i = 0; i < M; ++i) {
+            tbb::parallel_for(tbb::blocked_range<int>(0, N, 10), InnerBody(i));
+        }
+    }
+}
+/*end outer loop openmp with nested tbb*/
+
+void test() {
+    M = 2; N = 100;
+    TBB_NestedInOpenMP();
+}
+
+} // namespace nesting_tbb
+
+#ifndef _WIN32
+namespace pthreads_and_tbb {
+
+/*begin pthreads with tbb*/
+int M, N;
+
+struct InnerBody {
+    int i;
+    void operator()(tbb::blocked_range<int> const& r) const {
+        for (auto j = r.begin(); j != r.end(); ++j) {
+            // do the work for (i, j) element
+        }
+    }
+};
+
+void* OuterLoopIteration(void* args) {
+    int i = reinterpret_cast<intptr_t>(args);
+    tbb::parallel_for(tbb::blocked_range<int>(0, N, 10), InnerBody(i));
+    return nullptr;
+}
+
+void TBB_NestedInPThreads() {
+    std::vector<pthread_t> id(M);
+    // Create thread for each outer loop iteration
+    for(int i = 0; i < M; ++i) {
+        std::intptr_t arg = i;
+        pthread_create(&id[i], NULL, OuterLoopIteration, (void*)arg);
+    }
+    // Wait for outer loop threads to finish
+    for(int i = 0; i < M; ++i)
+        pthread_join(id[i], NULL);
+}
+/*end pthreads with tbb*/
+
+void test() {
+    M = 2; N = 100;
+    TBB_NestedInPThreads();
+}
+
+} // namespace pthreads_and_tbb
+#endif                          // _WIN32
+
+namespace nesting_omp {
+
+/*begin outer loop tbb with nested omp*/
+int M, N;
+
+void InnerBody(int i, int j) {
+    // do the work for (i, j) element
+}
+
+void OpenMP_NestedInTBB() {
+    tbb::parallel_for(0, M, [&](int i) {
+        #pragma omp parallel for
+        for(int j = 0; j < N; ++j) {
+            InnerBody(i, j);
+        }
+    });
+}
+/*end outer loop tbb with nested omp*/
+
+void test() {
+    M = 2; N = 100;
+    OpenMP_NestedInTBB();
+}
+
+} // namespace nesting_omp
+
+
+int main() {
+    // Setting maximum number of threads for both runtimes to avoid
+    // oversubscription issues
+    constexpr int max_threads = 2;
+    omp_set_num_threads(max_threads);
+    tbb::global_control gl(tbb::global_control::max_allowed_parallelism, max_threads);
+
+    nesting_tbb::test();
+#ifndef _WIN32
+    pthreads_and_tbb::test();
+#endif
+    nesting_omp::test();
+}