message: Fix a potential deadlock in GetOrCreateSubStream/ReturnSubStream by not holding Stream::mu_ while destroying bad streams.
diff: @@ -15,8 +15,6 @@ limitations under the License.
  #include "tensorflow/stream_executor/stream.h"
  -#include "tensorflow/stream_executor/platform/port.h"
  -
  #include "absl/strings/str_cat.h"
  #include "third_party/eigen3/Eigen/Core"
  #include "tensorflow/stream_executor/blas.h"
  @@ -24,6 +22,7 @@ limitations under the License.
  #include "tensorflow/stream_executor/lib/stacktrace.h"
  #include "tensorflow/stream_executor/platform.h"
  #include "tensorflow/stream_executor/platform/logging.h"
  +#include "tensorflow/stream_executor/platform/port.h"
  #include "tensorflow/stream_executor/rng.h"
  #include "tensorflow/stream_executor/stream_executor_internal.h"
  #include "tensorflow/stream_executor/stream_executor_pimpl.h"
  @@ -1874,6 +1873,10 @@ Stream &Stream::ThenMemcpyH2DQuantized(
  }
  Stream *Stream::GetOrCreateSubStream() {
  + // Do not destroy bad streams when holding mu_ because ~Stream() may
  + // BlockHostUntilDone and it's host callbacks might attempt to acquire mu_.
  + std::vector<std::unique_ptr<Stream>> bad_streams;
  +
  absl::MutexLock lock(&mu_);
  // Look for the first reusable sub_stream that is ok, dropping !ok sub_streams
  @@ -1897,6 +1900,7 @@ Stream *Stream::GetOrCreateSubStream() {
  if (index != last) {
  std::swap(pair, sub_streams_[last]);
  }
  + bad_streams.push_back(std::move(sub_streams_.back().first));
  sub_streams_.pop_back();
  VLOG(1) << DebugStreamPointers() << " dropped !ok sub_stream "
  << sub_stream->DebugStreamPointers();
  @@ -1921,6 +1925,10 @@ Stream *Stream::GetOrCreateSubStream() {
  }
  void Stream::ReturnSubStream(Stream *sub_stream) {
  + // Do not destroy bad streams when holding mu_ because ~Stream() may
  + // BlockHostUntilDone and it's host callbacks might attempt to acquire mu_.
  + std::unique_ptr<Stream> bad_stream;
  +
  absl::MutexLock lock(&mu_);
  // Look for the sub-stream.
  @@ -1945,6 +1953,7 @@ void Stream::ReturnSubStream(Stream *sub_stream) {
  if (index != last) {
  std::swap(pair, sub_streams_[last]);
  }
  + std::swap(bad_stream, sub_streams_.back().first);
  sub_streams_.pop_back();
  }
  return;
  