message: Eliminate test flakiness in fault_tolerance_test_v2.py with `-c opt` and `--copt=-UNDEBUG`.
diff: @@ -626,21 +626,30 @@ tensorflow::Status UpdateTFE_ContextWithServerDef(
  "targets will fail.";
  }
  } else {
  - // The master's context_view_id will be incremented by one
  - // the UpdateRemoteMaster call later. We want all new workers and
  - // existing workers to also have the updated context_view_id, so
  - // we must set their context_view_id to the existing master's
  - // context_view_id + 1.
  + if (sg.ok()) {
  + // Create remote contexts on the newly added workers only if the master
  + // has collected all device information from them (i.e., the
  + // GetAllRemoteDevices call returns succussfully). Note that in rare cases
  + // GetAllRemoteDevices can still fail even with RPCs configured to wait
  + // until the remote workers to become alive. If the master creates remote
  + // contexts on the workers whose devices are still not collected, those
  + // workers will be treated as existing workers subsequently, so the master
  + // will never get devices from them even with retrying UpdateServerDef.
  sg.Update(CreateRemoteContexts(
  ctx, added_workers, context_id, context_view_id + 1, keep_alive_secs,
  server_def, remote_eager_workers.get(), context->Executor().Async(),
  context->LazyCopyFunctionRemoteInputs(), base_request));
  + }
  if (!existing_workers.empty()) {
  if (VLOG_IS_ON(1)) {
  for (const string& w : existing_workers) {
  VLOG(1) << "Updating cluster with existing worker " << w;
  }
  }
  + // The master's context_view_id will be incremented by one in the
  + // UpdateRemoteMaster call later. We want existing workers to also have
  + // the updated context_view_id, so we must set their context_view_id to
  + // the master's current context_view_id + 1.
  sg.Update(UpdateRemoteContexts(ctx, existing_workers, added_workers,
  removed_workers, context_id,
  context_view_id + 1, server_def,
  @@ -847,20 +856,9 @@ TF_CAPI_EXPORT extern bool TFE_ContextCheckAlive(TFE_Context* ctx,
  #else // !defined(IS_MOBILE_PLATFORM)
  tensorflow::EagerContext* context =
  tensorflow::ContextFromInterface(tensorflow::unwrap(ctx));
  - tensorflow::GrpcServer* grpc_server =
  - static_cast<tensorflow::GrpcServer*>(context->GetServer());
  -
  - std::unique_ptr<tensorflow::eager::EagerClientCache> remote_eager_workers;
  - status->status = grpc_server->master_env()->worker_cache->GetEagerClientCache(
  - &remote_eager_workers);
  - if (!status->status.ok()) {
  - LOG(ERROR) << "Failed to get client cache for remote workers.";
  - return false;
  - }
  -
  // TODO(yuefengz): support partially specified `worker_name`.
  tensorflow::core::RefCountPtr<tensorflow::eager::EagerClient> eager_client;
  - status->status = remote_eager_workers->GetClient(worker_name, &eager_client);
  + status->status = context->GetClient(worker_name, &eager_client);
  if (!status->status.ok()) {
  return false;
  }
  