Revert "Merge pull request grpc#18146 from grpc/revert-17308-shutdown"

This reverts commit 9079e98, reversing
changes made to 76a38bf.

Author: yang-g

grpc.def

@@ -16,6 +16,7 @@ EXPORTS
     grpc_init
     grpc_shutdown
     grpc_is_initialized
+    grpc_shutdown_blocking
     grpc_version_string
     grpc_g_stands_for
     grpc_completion_queue_factory_lookup

include/grpc/grpc.h

@@ -73,10 +73,11 @@ GRPCAPI void grpc_init(void);
     Before it's called, there should haven been a matching invocation to
     grpc_init().
 
-    No memory is used by grpc after this call returns, nor are any instructions
-    executing within the grpc library.
-    Prior to calling, all application owned grpc objects must have been
-    destroyed. */
+    The last call to grpc_shutdown will initiate cleaning up of grpc library
+    internals, which can happen in another thread. Once the clean-up is done,
+    no memory is used by grpc, nor are any instructions executing within the
+    grpc library.  Prior to calling, all application owned grpc objects must
+    have been destroyed. */
 GRPCAPI void grpc_shutdown(void);
 
 /** EXPERIMENTAL. Returns 1 if the grpc library has been initialized.
@@ -85,6 +86,10 @@ GRPCAPI void grpc_shutdown(void);
     https://github.com/grpc/grpc/issues/15334 */
 GRPCAPI int grpc_is_initialized(void);
 
+/** EXPERIMENTAL. Blocking shut down grpc library.
+    This is only for wrapped language to use now. */
+GRPCAPI void grpc_shutdown_blocking(void);
+
 /** Return a string representing the current version of grpc */
 GRPCAPI const char* grpc_version_string(void);
 

src/core/lib/debug/trace.h

@@ -53,7 +53,8 @@ void grpc_tracer_enable_flag(grpc_core::TraceFlag* flag);
 class TraceFlag {
  public:
   TraceFlag(bool default_enabled, const char* name);
-  // This needs to be trivially destructible as it is used as global variable.
+  // TraceFlag needs to be trivially destructible since it is used as global
+  // variable.
   ~TraceFlag() = default;
 
   const char* name() const { return name_; }

src/core/lib/gprpp/thd.h

@@ -47,6 +47,27 @@ class ThreadInternalsInterface {
 
 class Thread {
  public:
+  class Options {
+   public:
+    Options() : joinable_(true), tracked_(true) {}
+    /// Set whether the thread is joinable or detached.
+    Options& set_joinable(bool joinable) {
+      joinable_ = joinable;
+      return *this;
+    }
+    bool joinable() const { return joinable_; }
+
+    /// Set whether the thread is tracked for fork support.
+    Options& set_tracked(bool tracked) {
+      tracked_ = tracked;
+      return *this;
+    }
+    bool tracked() const { return tracked_; }
+
+   private:
+    bool joinable_;
+    bool tracked_;
+  };
   /// Default constructor only to allow use in structs that lack constructors
   /// Does not produce a validly-constructed thread; must later
   /// use placement new to construct a real thread. Does not init mu_ and cv_
@@ -57,14 +78,17 @@ class Thread {
   /// with argument \a arg once it is started.
   /// The optional \a success argument indicates whether the thread
   /// is successfully created.
+  /// The optional \a options can be used to set the thread detachable.
   Thread(const char* thd_name, void (*thd_body)(void* arg), void* arg,
-         bool* success = nullptr);
+         bool* success = nullptr, const Options& options = Options());
 
   /// Move constructor for thread. After this is called, the other thread
   /// no longer represents a living thread object
-  Thread(Thread&& other) : state_(other.state_), impl_(other.impl_) {
+  Thread(Thread&& other)
+      : state_(other.state_), impl_(other.impl_), options_(other.options_) {
     other.state_ = MOVED;
     other.impl_ = nullptr;
+    other.options_ = Options();
   }
 
   /// Move assignment operator for thread. After this is called, the other
@@ -79,8 +103,10 @@ class Thread {
       // assert it for the time being.
       state_ = other.state_;
       impl_ = other.impl_;
+      options_ = other.options_;
       other.state_ = MOVED;
       other.impl_ = nullptr;
+      other.options_ = Options();
     }
     return *this;
   }
@@ -95,11 +121,16 @@ class Thread {
       GPR_ASSERT(state_ == ALIVE);
       state_ = STARTED;
       impl_->Start();
+      if (!options_.joinable()) {
+        state_ = DONE;
+        impl_ = nullptr;
+      }
     } else {
       GPR_ASSERT(state_ == FAILED);
     }
-  };
+  }
 
+  // It is only legal to call Join if the Thread is created as joinable.
   void Join() {
     if (impl_ != nullptr) {
       impl_->Join();
@@ -119,12 +150,13 @@ class Thread {
   /// FAKE -- just a dummy placeholder Thread created by the default constructor
   /// ALIVE -- an actual thread of control exists associated with this thread
   /// STARTED -- the thread of control has been started
-  /// DONE -- the thread of control has completed and been joined
+  /// DONE -- the thread of control has completed and been joined/detached
   /// FAILED -- the thread of control never came alive
   /// MOVED -- contents were moved out and we're no longer tracking them
   enum ThreadState { FAKE, ALIVE, STARTED, DONE, FAILED, MOVED };
   ThreadState state_;
   internal::ThreadInternalsInterface* impl_;
+  Options options_;
 };
 
 }  // namespace grpc_core

src/core/lib/gprpp/thd_posix.cc

@@ -44,13 +44,14 @@ struct thd_arg {
   void (*body)(void* arg); /* body of a thread */
   void* arg;               /* argument to a thread */
   const char* name;        /* name of thread. Can be nullptr. */
+  bool joinable;
+  bool tracked;
 };
 
-class ThreadInternalsPosix
-    : public grpc_core::internal::ThreadInternalsInterface {
+class ThreadInternalsPosix : public internal::ThreadInternalsInterface {
  public:
   ThreadInternalsPosix(const char* thd_name, void (*thd_body)(void* arg),
-                       void* arg, bool* success)
+                       void* arg, bool* success, const Thread::Options& options)
       : started_(false) {
     gpr_mu_init(&mu_);
     gpr_cv_init(&ready_);
@@ -63,11 +64,20 @@ class ThreadInternalsPosix
     info->body = thd_body;
     info->arg = arg;
     info->name = thd_name;
-    grpc_core::Fork::IncThreadCount();
+    info->joinable = options.joinable();
+    info->tracked = options.tracked();
+    if (options.tracked()) {
+      Fork::IncThreadCount();
+    }
 
     GPR_ASSERT(pthread_attr_init(&attr) == 0);
-    GPR_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE) ==
-               0);
+    if (options.joinable()) {
+      GPR_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE) ==
+                 0);
+    } else {
+      GPR_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) ==
+                 0);
+    }
 
     *success =
         (pthread_create(&pthread_id_, &attr,
@@ -97,8 +107,14 @@ class ThreadInternalsPosix
                           }
                           gpr_mu_unlock(&arg.thread->mu_);
 
+                          if (!arg.joinable) {
+                            Delete(arg.thread);
+                          }
+
                           (*arg.body)(arg.arg);
-                          grpc_core::Fork::DecThreadCount();
+                          if (arg.tracked) {
+                            Fork::DecThreadCount();
+                          }
                           return nullptr;
                         },
                         info) == 0);
@@ -108,9 +124,11 @@ class ThreadInternalsPosix
     if (!(*success)) {
       /* don't use gpr_free, as this was allocated using malloc (see above) */
       free(info);
-      grpc_core::Fork::DecThreadCount();
+      if (options.tracked()) {
+        Fork::DecThreadCount();
+      }
     }
-  };
+  }
 
   ~ThreadInternalsPosix() override {
     gpr_mu_destroy(&mu_);
@@ -136,15 +154,15 @@ class ThreadInternalsPosix
 }  // namespace
 
 Thread::Thread(const char* thd_name, void (*thd_body)(void* arg), void* arg,
-               bool* success) {
+               bool* success, const Options& options)
+    : options_(options) {
   bool outcome = false;
-  impl_ =
-      grpc_core::New<ThreadInternalsPosix>(thd_name, thd_body, arg, &outcome);
+  impl_ = New<ThreadInternalsPosix>(thd_name, thd_body, arg, &outcome, options);
   if (outcome) {
     state_ = ALIVE;
   } else {
     state_ = FAILED;
-    grpc_core::Delete(impl_);
+    Delete(impl_);
     impl_ = nullptr;
   }
 

src/core/lib/gprpp/thd_windows.cc

@@ -46,14 +46,16 @@ struct thd_info {
   void (*body)(void* arg); /* body of a thread */
   void* arg;               /* argument to a thread */
   HANDLE join_event;       /* the join event */
+  bool joinable;           /* whether it is joinable */
 };
 
 thread_local struct thd_info* g_thd_info;
 
 class ThreadInternalsWindows
     : public grpc_core::internal::ThreadInternalsInterface {
  public:
-  ThreadInternalsWindows(void (*thd_body)(void* arg), void* arg, bool* success)
+  ThreadInternalsWindows(void (*thd_body)(void* arg), void* arg, bool* success,
+                         const grpc_core::Thread::Options& options)
       : started_(false) {
     gpr_mu_init(&mu_);
     gpr_cv_init(&ready_);
@@ -63,21 +65,24 @@ class ThreadInternalsWindows
     info_->thread = this;
     info_->body = thd_body;
     info_->arg = arg;
-
-    info_->join_event = CreateEvent(nullptr, FALSE, FALSE, nullptr);
-    if (info_->join_event == nullptr) {
-      gpr_free(info_);
-      *success = false;
-    } else {
-      handle = CreateThread(nullptr, 64 * 1024, thread_body, info_, 0, nullptr);
-      if (handle == nullptr) {
-        destroy_thread();
+    info_->join_event = nullptr;
+    info_->joinable = options.joinable();
+    if (info_->joinable) {
+      info_->join_event = CreateEvent(nullptr, FALSE, FALSE, nullptr);
+      if (info_->join_event == nullptr) {
+        gpr_free(info_);
         *success = false;
-      } else {
-        CloseHandle(handle);
-        *success = true;
+        return;
       }
     }
+    handle = CreateThread(nullptr, 64 * 1024, thread_body, info_, 0, nullptr);
+    if (handle == nullptr) {
+      destroy_thread();
+      *success = false;
+    } else {
+      CloseHandle(handle);
+      *success = true;
+    }
   }
 
   ~ThreadInternalsWindows() override {
@@ -107,14 +112,24 @@ class ThreadInternalsWindows
                   gpr_inf_future(GPR_CLOCK_MONOTONIC));
     }
     gpr_mu_unlock(&g_thd_info->thread->mu_);
+    if (!g_thd_info->joinable) {
+      grpc_core::Delete(g_thd_info->thread);
+      g_thd_info->thread = nullptr;
+    }
     g_thd_info->body(g_thd_info->arg);
-    BOOL ret = SetEvent(g_thd_info->join_event);
-    GPR_ASSERT(ret);
+    if (g_thd_info->joinable) {
+      BOOL ret = SetEvent(g_thd_info->join_event);
+      GPR_ASSERT(ret);
+    } else {
+      gpr_free(g_thd_info);
+    }
     return 0;
   }
 
   void destroy_thread() {
-    CloseHandle(info_->join_event);
+    if (info_ != nullptr && info_->joinable) {
+      CloseHandle(info_->join_event);
+    }
     gpr_free(info_);
   }
 
@@ -129,14 +144,15 @@ class ThreadInternalsWindows
 namespace grpc_core {
 
 Thread::Thread(const char* thd_name, void (*thd_body)(void* arg), void* arg,
-               bool* success) {
+               bool* success, const Options& options)
+    : options_(options) {
   bool outcome = false;
-  impl_ = grpc_core::New<ThreadInternalsWindows>(thd_body, arg, &outcome);
+  impl_ = New<ThreadInternalsWindows>(thd_body, arg, &outcome, options);
   if (outcome) {
     state_ = ALIVE;
   } else {
     state_ = FAILED;
-    grpc_core::Delete(impl_);
+    Delete(impl_);
     impl_ = nullptr;
   }