TensorFlow hangs in session.run
In LLDB where it hangs:

<details>

```
(lldb) thread backtrace all
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
  * frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0e7954 libtensorflow_framework.2.dylib`nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) + 412
    frame #4: 0x000000014b0e42f8 libtensorflow_framework.2.dylib`nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) + 372
    frame #5: 0x0000000284afeb9c _pywrap_tensorflow_internal.so`tensorflow::DirectSession::WaitForNotification(tsl::Notification*, long long) + 216
    frame #6: 0x0000000284af6c38 _pywrap_tensorflow_internal.so`tensorflow::DirectSession::WaitForNotification(tsl::Notification*, tensorflow::DirectSession::RunState*, tsl::CancellationManager*, long long) + 44
    frame #7: 0x0000000284af5a90 _pywrap_tensorflow_internal.so`tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) + 2752
    frame #8: 0x0000000284af76e8 _pywrap_tensorflow_internal.so`tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor> > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<tensorflow::Tensor, std::__1::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) + 1800
    frame #9: 0x0000000284af6fd4 _pywrap_tensorflow_internal.so`tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor> > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<tensorflow::Tensor, std::__1::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) + 32
    frame #10: 0x000000028098ca60 _pywrap_tensorflow_internal.so`tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor> > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<tensorflow::Tensor, std::__1::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) + 248
    frame #11: 0x00000002813cf550 _pywrap_tensorflow_internal.so`TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, tensorflow::Tensor> > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, TF_Tensor**, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, TF_Buffer*, TF_Status*) + 508
    frame #12: 0x00000002813db464 _pywrap_tensorflow_internal.so`TF_SessionRun + 972
    frame #13: 0x000000028098aa38 _pywrap_tensorflow_internal.so`tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<_object*, std::__1::allocator<_object*> > const&, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<TF_Operation*, std::__1::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::__1::vector<_object*, std::__1::allocator<_object*> >*) + 816
    frame #14: 0x000000028098aeb4 _pywrap_tensorflow_internal.so`tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<_object*, std::__1::allocator<_object*> > const&, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<TF_Operation*, std::__1::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::__1::vector<_object*, std::__1::allocator<_object*> >*) + 56
    frame #15: 0x000000014a3bb044 _pywrap_tf_session.so`void pybind11::cpp_function::initialize<pybind11_init__pywrap_tf_session(pybind11::module_&)::$_16, pybind11::object, TF_Session*, TF_Buffer*, pybind11::handle const&, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<TF_Operation*, std::__1::allocator<TF_Operation*> > const&, TF_Buffer*, pybind11::name, pybind11::scope, pybind11::sibling>(pybind11_init__pywrap_tf_session(pybind11::module_&)::$_16&&, pybind11::object (*)(TF_Session*, TF_Buffer*, pybind11::handle const&, std::__1::vector<TF_Output, std::__1::allocator<TF_Output> > const&, std::__1::vector<TF_Operation*, std::__1::allocator<TF_Operation*> > const&, TF_Buffer*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&)::'lambda'(pybind11::detail::function_call&)::__invoke(pybind11::detail::function_call&) + 648
    frame #16: 0x000000014a39d010 _pywrap_tf_session.so`pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 3580
    frame #17: 0x00000001000b3398 python3`cfunction_call + 80
    frame #18: 0x000000010005f1e8 python3`_PyObject_MakeTpCall + 340
    frame #19: 0x000000010016f6ac python3`call_function + 724
    frame #20: 0x000000010016bd44 python3`_PyEval_EvalFrameDefault + 29268
    frame #21: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #22: 0x000000010016f614 python3`call_function + 572
    frame #23: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #24: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #25: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #26: 0x000000010016c078 python3`_PyEval_EvalFrameDefault + 30088
    frame #27: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #28: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #29: 0x000000010016f614 python3`call_function + 572
    frame #30: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #31: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #32: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #33: 0x000000010016f614 python3`call_function + 572
    frame #34: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #35: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #36: 0x000000010016f614 python3`call_function + 572
    frame #37: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #38: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #39: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #40: 0x0000000100062cf0 python3`method_vectorcall + 164
    frame #41: 0x000000010016f614 python3`call_function + 572
    frame #42: 0x000000010016be40 python3`_PyEval_EvalFrameDefault + 29520
    frame #43: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #44: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #45: 0x0000000100062cf0 python3`method_vectorcall + 164
    frame #46: 0x000000010016f614 python3`call_function + 572
    frame #47: 0x000000010016be40 python3`_PyEval_EvalFrameDefault + 29520
    frame #48: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #49: 0x000000010016f614 python3`call_function + 572
    frame #50: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #51: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #52: 0x000000010016f614 python3`call_function + 572
    frame #53: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #54: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #55: 0x000000010016f614 python3`call_function + 572
    frame #56: 0x000000010016bdc4 python3`_PyEval_EvalFrameDefault + 29396
    frame #57: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #58: 0x000000010016f614 python3`call_function + 572
    frame #59: 0x000000010016bdc4 python3`_PyEval_EvalFrameDefault + 29396
    frame #60: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #61: 0x00000001001c7834 python3`pyrun_file + 376
    frame #62: 0x00000001001c6d48 python3`PyRun_SimpleFileExFlags + 816
    frame #63: 0x00000001001e9e84 python3`Py_RunMain + 2916
    frame #64: 0x00000001001eb018 python3`pymain_main + 1272
    frame #65: 0x0000000100005ddc python3`main + 56
    frame #66: 0x00000001003d108c dyld`start + 520
  thread #11
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47e7d4 _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 576
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #12
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #13
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #14
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #15
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #16
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #17
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #18
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #19
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #20
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #21
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #22
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #23
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #24
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #25
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #26
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #27
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000028a47f320 _pywrap_tensorflow_internal.so`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000028a47f03c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000028a47eb4c _pywrap_tensorflow_internal.so`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000028a47e468 _pywrap_tensorflow_internal.so`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #28
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce490 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 576
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #29
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce808 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1464
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #30
    frame #0: 0x000000018382a06c libsystem_kernel.dylib`__semwait_signal + 8
    frame #1: 0x0000000183732fc8 libsystem_c.dylib`nanosleep + 220
    frame #2: 0x000000014bb9d7f4 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PosixEnv::SleepForMicroseconds(long long) + 152
    frame #3: 0x000000014bcfb620 libtensorflow_framework.2.dylib`tensorflow::EventMgr::PollLoop() + 500
    frame #4: 0x000000014b0ce828 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 1496
    frame #5: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #6: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #7: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #31
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001837b3294 libc++.1.dylib`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 28
    frame #3: 0x000000014b0cefdc libtensorflow_framework.2.dylib`Eigen::EventCount::CommitWait(Eigen::EventCount::Waiter*) + 208
    frame #4: 0x000000014b0cecf8 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) + 952
    frame #5: 0x000000014b0ce490 libtensorflow_framework.2.dylib`Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) + 576
    frame #6: 0x000000014b0ce124 libtensorflow_framework.2.dylib`tsl::thread::EigenEnvironment::CreateThread(std::__1::function<void ()>)::'lambda'()::operator()() const + 80
    frame #7: 0x000000014bb9e878 libtensorflow_framework.2.dylib`tsl::(anonymous namespace)::PThread::ThreadFn(void*) + 120
    frame #8: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #32
    frame #0: 0x000000018382a270 libsystem_kernel.dylib`__psynch_cvwait + 8
    frame #1: 0x000000018386483c libsystem_pthread.dylib`_pthread_cond_wait + 1236
    frame #2: 0x00000001001d96a0 python3`PyThread_acquire_lock_timed + 604
    frame #3: 0x00000001002478e4 python3`acquire_timed + 236
    frame #4: 0x0000000100247aa8 python3`lock_PyThread_acquire_lock + 72
    frame #5: 0x000000010006d284 python3`method_vectorcall_VARARGS_KEYWORDS + 292
    frame #6: 0x000000010016f614 python3`call_function + 572
    frame #7: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #8: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #9: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #10: 0x000000010016f614 python3`call_function + 572
    frame #11: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #12: 0x00000001001644a8 python3`_PyEval_EvalCode + 2968
    frame #13: 0x000000010005fe64 python3`_PyFunction_Vectorcall + 240
    frame #14: 0x000000010016f614 python3`call_function + 572
    frame #15: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #16: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #17: 0x0000000100062d9c python3`method_vectorcall + 336
    frame #18: 0x000000010016c078 python3`_PyEval_EvalFrameDefault + 30088
    frame #19: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #20: 0x000000010016f614 python3`call_function + 572
    frame #21: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #22: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #23: 0x000000010016f614 python3`call_function + 572
    frame #24: 0x000000010016bd28 python3`_PyEval_EvalFrameDefault + 29240
    frame #25: 0x000000010005fee4 python3`function_code_fastcall + 116
    frame #26: 0x0000000100062d9c python3`method_vectorcall + 336
    frame #27: 0x0000000100246d50 python3`t_bootstrap + 180
    frame #28: 0x00000001001d91c8 python3`pythread_wrapper + 48
    frame #29: 0x000000018386426c libsystem_pthread.dylib`_pthread_start + 148
  thread #33
    frame #0: 0x000000018382872c libsystem_kernel.dylib`__workq_kernreturn + 8
```

</details> When executing on CPU, I get an actual exception:
```
TensorFlow exception: Graph execution error:

indices[1] = 7 is not in [0, 7)
	 [[{{node output/rec/enc_value_gather/GatherV2}}]]
```
And this is probably correct. It's very possible that I have the index off for `tf.gather`. I think I did not do any bound checks.

So maybe that is the error here on GPU? Maybe there are no bound checks for `tf.gather` on GPU, and this somehow causes it to access some memory region which hangs?
**Edit** Not sure. I cannot really trigger any strange behavior manually by trial and error using `tf.gather` and invalid indices on GPU. It always returns 0. I assume this is still safe and the problem is elsewhere.
**Edit** I explicitly clipped the values now in all cases, and I still have the same hang, so this seems to be unrelated. @albertz I have tried to reproduce this error in colab on TF 2.11 and did not face any error. The GPU was enabled during the experiment. Please find the gist [here](https://colab.research.google.com/gist/pjpratik/175321e781bacf4dc0540e7b44dea00e/59722.ipynb). 

Have you checked in the latest TF version and see if the issue still exists? Thanks! Thanks for creating the gist.

However, as I said, it is crucial to run it on Apple M1 hardware to reproduce it. In your gist, I see that you use an Nvidia GPU.

Also, you did not exactly use the commits I specified, although this probably should not matter.
 @albertz Thanks for the clarification. I was able to reproduce the error with the commits on Apple M1 hardware. The code runs fine else wise. 

**Hangs here when the specific commits are used**
<img width="890" alt="Screenshot 2023-02-20 at 5 25 09 PM" src="https://user-images.githubusercontent.com/118897289/220099655-20e8f0b2-f303-454f-92c7-a0b4afb2dc6a.png">

**Works fine else wise**
<img width="693" alt="Screenshot 2023-02-20 at 4 27 50 PM" src="https://user-images.githubusercontent.com/118897289/220086396-c2ae8b7f-9f7c-480c-924e-ffcf1efba09b.png">


Hi @gaikwadrahul8 Could you please look into this issue. Thanks. Hi, @albertz 

Apologize for the delay and I was able to reproduce the issue with specific commits on `Apple M1` and it seems like Tensorflow hangs in session.run() or else it's working fine as expected, For you reference I have added screenshots below and as per [official documentation](https://www.tensorflow.org/install/pip#macos), **There is currently no official GPU support for MacOS.** It's better to post this issue [here](https://github.com/apple/tensorflow_macos/issues) for fast resolution. Thank you!

Working fine as expected without specific commits :

```
dev: score 0.3890726839668453 error 0.1398488128509993
train epoch 100, finished after 96 steps, 0:00:11 elapsed (99.5% computing time)                                                                                                                                      
epoch 100 score: 0.3872191429890062 error: 0.13927021657019117 elapsed: 0:00:11
epoch 100 'dev' eval, finished after 10 steps, 0:00:00 elapsed (98.9% computing time)                                                                                                                                 
dev: score 0.4133371474361815 error 0.1389488848403751
Finished training in epoch 100.
elapsed: 0:21:43.9778
```

Here is screenshot where Tensorflow hangs with specific commits:

```
2023-02-24 19:59:43.993909: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.
2023-02-24 19:59:44.001514: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.
Create optimizer <class 'tensorflow.python.training.adam.AdamOptimizer'> with options {'epsilon': 1e-08, 'learning_rate': <tf.Variable 'learning_rate:0' shape=() dtype=float32>}.
Initialize optimizer (default) with slots ['m', 'v'].
These additional variable were created by the optimizer: [<tf.Variable 'optimize/beta1_power:0' shape=() dtype=float32>, <tf.Variable 'optimize/beta2_power:0' shape=() dtype=float32>].
2023-02-24 19:59:44.714013: W tensorflow/c/c_api.cc:291] Operation '{name:'global_step' id:1528 op device:{requested: '/device:CPU:0', assigned: ''} def:{{{node global_step}} = VarHandleOp[_class=["loc:@global_step"], _has_manual_control_dependencies=true, allowed_devices=[], container="", dtype=DT_INT64, shape=[], shared_name="global_step", _device="/device:CPU:0"]()}}' was changed by setting attribute after it was run by a session. This mutation will have no effect, and will trigger an error in the future. Either don't modify nodes after running them or create a new session.
2023-02-24 19:59:44.757001: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.
2023-02-24 19:59:44.855683: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.
2023-02-24 19:59:44.954833: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.
``` > **There is currently no official GPU support for MacOS.** It's better to post this issue [here](https://github.com/apple/tensorflow_macos/issues) for fast resolution. Thank you!

It says:

> This repository has been archived by the owner on Jun 10, 2021. It is now read-only.

I cannot post any new issue there.
 Hi, @albertz 

You can post this issue on [Apple Developer Forums ](https://developer.apple.com/forums/tags/tensorflow-metal)for fast resolution. Thank you! I posted it here: https://developer.apple.com/forums/thread/725543 Hi, @albertz 

Thank you for posting this issue to [Apple Developer Forums ](https://developer.apple.com/forums/tags/tensorflow-metal) and it will be taken care by Apple Developer Team so Could you please close this issue because There is currently no official GPU support for MacOS from our end? Thank you! Are you satisfied with the resolution of your issue?
<a href="https://docs.google.com/forms/d/e/1FAIpQLSfaP12TRhd9xSxjXZjcZFNXPGk4kc1-qMdv3gc6bEP90vY1ew/viewform?entry.85265664=Yes&entry.2137816233=https://github.com/tensorflow/tensorflow/issues/59722">Yes</a>
<a href="https://docs.google.com/forms/d/e/1FAIpQLSfaP12TRhd9xSxjXZjcZFNXPGk4kc1-qMdv3gc6bEP90vY1ew/viewform?entry.85265664=No&entry.2137816233=https://github.com/tensorflow/tensorflow/issues/59722">No</a>

<details><summary>Click to expand!</summary> 
 
 ### Issue Type

Bug

### Have you reproduced the bug with TF nightly?

No

### Source

binary

### Tensorflow Version

tensorflow-macos 2.11.0

### Custom Code

Yes

### OS Platform and Distribution

MacOS 12, tensorflow-macos and tensorflow-metal

### Mobile device

_No response_

### Python version

3.9

### Bazel version

_No response_

### GCC/Compiler version

_No response_

### CUDA/cuDNN version

_No response_

### GPU model and memory

M1

</details>

### Current Behaviour?

This is a new neural model I implemented, and I want to do training. It's modified based on an existing attention-based encoder-decoder model, where everything works fine.

In the new model, it just hangs in `session.run` and does not do anything. I also cannot interrupt it. It hangs inside the TensorFlow C++ code.

This seems to be specific for Mac M1 hardware. I cannot reproduce the problem on other hardware or environments.


### Standalone code to reproduce the issue

So far I don't have a minimal example, and this will be quite a big effort to generate one, as this is some very complex model. But here some relevant details:

* This is based on [RETURNN](https://github.com/rwth-i6/returnn).
* We still use graph-mode.
* I tested both with control flow v1 (calling `disable_control_flow_v2`) and control flow v2. It hangs in both cases.
* I tested using tfdbg or `enable_dump_debug_info`. It crashes then with a segfault.
* I get a number of other warnings, which are maybe related. See below.

To reproduce:

* Code: https://github.com/rwth-i6/i6_experiments/blob/81bcef39b5829aa43b84bcab4b4fa03f82fc3bc5/users/zeyer/experiments/exp2023_02_16_chunked_attention/demo_returnn_config.py
* Checkout the [i6_experiments repo](https://github.com/rwth-i6/i6_experiments). (Maybe use commit 81bcef39b5829aa43b84bcab4b4fa03f82fc3bc5 to be sure.)
* Checkout [RETURNN](https://github.com/rwth-i6/returnn). (Maybe commit 2ed598443f22de42599a0fee9bc43fbb5e0abec2 to be sure.)
* Run: `python3 returnn/rnn.py i6_experiments/users/zeyer/experiments/exp2023_02_16_chunked_attention/demo_returnn_config.py`


### Relevant log output

With control flow v2:

```
2023-02-17 10:02:03.997491: W tensorflow/core/common_runtime/type_inference.cc:339] Type inference failed. This indicates an invalid graph that escaped type checking. Error message: INVALID_ARGUMENT: expected compatible input types, but input 1:
type_id: TFT_OPTIONAL
args {
  type_id: TFT_PRODUCT
  args {
    type_id: TFT_TENSOR
    args {
      type_id: TFT_INT32
    }
  }
}
 is neither a subtype nor a supertype of the combined inputs preceding it:
type_id: TFT_OPTIONAL
args {
  type_id: TFT_PRODUCT
  args {
    type_id: TFT_TENSOR
    args {
      type_id: TFT_FLOAT
    }
  }
}

	while inferring type of node 'output/rec/while/body/_38/output/rec/prev_target_embed_moved_input/cond/output/_1608'

2023-02-17 10:34:46.595736: W tensorflow/c/c_api.cc:291] Operation '{name:'global_step' id:1961 op device:{requested: '/device:CPU:0', assigned: ''} def:{{{node global_step}} = VarHandleOp[_class=["loc:@global_step"], _has_manual_control_dependencies=true, allowed_devices=[], container="", dtype=DT_INT64, shape=[], shared_name="global_step", _device="/device:CPU:0"]()}}' was changed by setting attribute after it was run by a session. This mutation will have no effect, and will trigger an error in the future. Either don't modify nodes after running them or create a new session.
2023-02-17 10:35:56.799620+0100 python3[5197:2744697] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
...
2023-02-17 10:36:01.801307+0100 python3[5197:2744697] Execution of the command buffer was aborted due to an error during execution. Ignored (for causing prior/excessive GPU errors) (00000004:kIOGPUCommandBufferCallbackErrorSubmissionsIgnored)
...
```
(Related: https://github.com/tensorflow/tensorflow/issues/57052)

With control flow v1:
```
2023-02-17 10:10:01.733679: W tensorflow/c/c_api.cc:291] Operation '{name:'global_step' id:1528 op device:{requested: '/device:CPU:0', assigned: ''} def:{{{node global_step}} = VarHandleOp[_class=["loc:@global_step"], _has_manual_control_dependencies=true, allowed_devices=[], container="", dtype=DT_INT64, shape=[], shared_name="global_step", _device="/device:CPU:0"]()}}' was changed by setting attribute after it was run by a session. This mutation will have no effect, and will trigger an error in the future. Either don't modify nodes after running them or create a new session.
2023-02-17 10:10:14.257716+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:14.257754+0100 python3[3727:2732582] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:14.258366+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:14.258504+0100 python3[3727:2732582] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:14.258541+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:14.258587+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:19.258726+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
2023-02-17 10:10:19.258784+0100 python3[3727:2732395] Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
```
