SCBert/SCBert.py
def forward_and_pool (self, input_ids_tensor, masks_tensor, sentence_pooling_method="average", word_pooling_method="average", layers = 11, batch_size=50, path_to_save=None) :
            """
            This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods. 
            
            Parameters
            ----------
            input_ids_tensor : tensor
                Corresponds to then= ids of tokenized words. 
                Must match the output of the tokenize function.
            masks_tensor : tensor
                Corresponds to the attention masks of tokenized words. 
                Must match the output of the tokenize function. .
            sentence_pooling_method : str, optional
                Corresponds to the method of pooling to create a unique vector for each text
                The default is "average".
            word_pooling_method : str, optional
                Corresponds to the method of word pooling method in the case multiple layers have been stated. In other words it is the way to compute a single word vector from multiple layers.
                The default is "average".
            layers : int or list, optional
                Corresponds to the BERT layers to use to create the embeddings of words.
                The default is 11.
            batch_size : int, optional
                Size of batch when executing the forward pass into the BERT model, should get lower as your computational power gets lower. 
                The default is 50.
            path_to_save : str or None, optional
                Is the path to save the vector of texts if you want to avoid doing this computation again as it may be long. 
                if None, nothing will be saved. 
                The default is None.
            
            Returns
            -------
            texts_vectors : list
                A list of tensors, each tensor corresponding to an input text.

            """
            
            layer_list = False
            if (sentence_pooling_method not in ["average", "max"]) :
                raise ValueError('sentence_pooling_method must be equal to `average` or `max`')
                
            if (word_pooling_method not in ["average", "max", "concat"]) :
                raise ValueError('word_pooling_method must be equal to `average`, `max` or `concat` ')
            
            if(type(batch_size) != int) :
                raise TypeError('batch_size must be a positive integer')
                
            if(batch_size<=0) :
                raise ValueError('batch_size must be a positive integer')
                
            if((type(path_to_save)  != str ) and (path_to_save != None)):
                raise TypeError('path_to_save must be None or a string')
                
            if (type(layers) != int) :
                if (type(layers) == list) :
                    layer_list = True
                    for el in layers : 
                        if (type(el) != int) :
                            raise TypeError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
                        elif (el>self.nb_layer or el<1) :
                            raise ValueError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
                else :
                    raise TypeError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
            else :
                if (layers>self.nb_layer or layers<1) :
                    raise ValueError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
            
            texts_vectors = []
            N = len(input_ids_tensor)
            counter = 0
            with tqdm(total = 100) as pbar : 
                for b in self.__batch(range(0,N), batch_size) :
                    with torch.no_grad() :
                        encoded_layers_b = self.model(input_ids_tensor[b], masks_tensor[b].to(torch.int64))[1]
                            
                        if layer_list :
                            for idx in b :
                                if input_ids_tensor[idx][-1]==1 :
                                    eos_pos = 0
                                else :
                                    eos_pos = int((input_ids_tensor[idx] == self.pad_id).nonzero()[0])
                                word_vector = self.__word_pooling(encoded_layers_b, layers, idx - counter, word_pooling_method)
                                pooled_vector = self.__sentence_pooling(word_vector[:eos_pos-1][1:], sentence_pooling_method) #Just no to take into account BOS and EOS 
                                texts_vectors.append(pooled_vector)
                            
                        else : 
                            words_vector = encoded_layers_b[layers]
                            
                            for idx in b : 
                                if input_ids_tensor[idx][-1]==1 :
                                    eos_pos = 0
                                else :
                                    eos_pos = int((input_ids_tensor[idx] == self.pad_id).nonzero()[0])
                                pooled_vector = self.__sentence_pooling(words_vector[idx-counter][:eos_pos-1][1:], sentence_pooling_method) #Just no to take into account BOS and EOS 
                                texts_vectors.append(pooled_vector)
                    counter += batch_size
                    pbar.update(np.round(100*len(b)/N,2))
            
            if path_to_save != None : 
              torch.save(texts_vectors, path_to_save+"text_vectors.pt")
            
            return texts_vectors

            This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods. 
            
            Parameters
            ----------
            input_ids_tensor : tensor
                Corresponds to then= ids of tokenized words. 
                Must match the output of the tokenize function.
            masks_tensor : tensor
                Corresponds to the attention masks of tokenized words. 
                Must match the output of the tokenize function. .
            sentence_pooling_method : str, optional
                Corresponds to the method of pooling to create a unique vector for each text
                The default is "average".
            word_pooling_method : str, optional
                Corresponds to the method of word pooling method in the case multiple layers have been stated. In other words it is the way to compute a single word vector from multiple layers.
                The default is "average".
            layers : int or list, optional
                Corresponds to the BERT layers to use to create the embeddings of words.
                The default is 11.
            batch_size : int, optional
                Size of batch when executing the forward pass into the BERT model, should get lower as your computational power gets lower. 
                The default is 50.
            path_to_save : str or None, optional
                Is the path to save the vector of texts if you want to avoid doing this computation again as it may be long. 
                if None, nothing will be saved. 
                The default is None.
            
            Returns
            -------
            texts_vectors : list
                A list of tensors, each tensor corresponding to an input text.

            
This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods.
Parameters

input_ids_tensor : tensor
Corresponds to then= ids of tokenized words.
Must match the output of the tokenize function.
masks_tensor : tensor
Corresponds to the attention masks of tokenized words.
Must match the output of the tokenize function.

Returns

texts_vectors : list
A list of tensors, each tensor corresponding to an input text.
This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods . Parameters input_ids_tensor : tensor Corresponds to then = ids of tokenized words . Must match the output of the tokenize function . masks_tensor : tensor Corresponds to the attention masks of tokenized words . Must match the output of the tokenize function . Returns texts_vectors : list A list of tensors each tensor corresponding to an input text .
def forward_and_pool (self, input_ids_tensor, masks_tensor, sentence_pooling_method="average", word_pooling_method="average", layers = 11, batch_size=50, path_to_save=None) :
            layer_list = False
            if (sentence_pooling_method not in ["average", "max"]) :
                raise ValueError('sentence_pooling_method must be equal to `average` or `max`')
            if (word_pooling_method not in ["average", "max", "concat"]) :
                raise ValueError('word_pooling_method must be equal to `average`, `max` or `concat` ')
            if(type(batch_size) != int) :
                raise TypeError('batch_size must be a positive integer')
            if(batch_size<=0) :
                raise ValueError('batch_size must be a positive integer')
            if((type(path_to_save)  != str ) and (path_to_save != None)):
                raise TypeError('path_to_save must be None or a string')
            if (type(layers) != int) :
                if (type(layers) == list) :
                    layer_list = True
                    for el in layers : 
                        if (type(el) != int) :
                            raise TypeError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
                        elif (el>self.nb_layer or el<1) :
                            raise ValueError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
                else :
                    raise TypeError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
            else :
                if (layers>self.nb_layer or layers<1) :
                    raise ValueError('layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer')
            texts_vectors = []
            N = len(input_ids_tensor)
            counter = 0
            with tqdm(total = 100) as pbar : 
                for b in self.__batch(range(0,N), batch_size) :
                    with torch.no_grad() :
                        encoded_layers_b = self.model(input_ids_tensor[b], masks_tensor[b].to(torch.int64))[1]
                        if layer_list :
                            for idx in b :
                                if input_ids_tensor[idx][-1]==1 :
                                    eos_pos = 0
                                else :
                                    eos_pos = int((input_ids_tensor[idx] == self.pad_id).nonzero()[0])
                                word_vector = self.__word_pooling(encoded_layers_b, layers, idx - counter, word_pooling_method)
                                pooled_vector = self.__sentence_pooling(word_vector[:eos_pos-1][1:], sentence_pooling_method) 
                                texts_vectors.append(pooled_vector)
                        else : 
                            words_vector = encoded_layers_b[layers]
                            for idx in b : 
                                if input_ids_tensor[idx][-1]==1 :
                                    eos_pos = 0
                                else :
                                    eos_pos = int((input_ids_tensor[idx] == self.pad_id).nonzero()[0])
                                pooled_vector = self.__sentence_pooling(words_vector[idx-counter][:eos_pos-1][1:], sentence_pooling_method) 
                                texts_vectors.append(pooled_vector)
                    counter += batch_size
                    pbar.update(np.round(100*len(b)/N,2))
            if path_to_save != None : 
              torch.save(texts_vectors, path_to_save+"text_vectors.pt")
            return texts_vectors
def forward_and_pool ( self , input_ids_tensor , masks_tensor , sentence_pooling_method = "average" , word_pooling_method = "average" , layers = 11 , batch_size = 50 , path_to_save = None ) : layer_list = False if ( sentence_pooling_method not in [ "average" , "max" ] ) : raise ValueError ( 'sentence_pooling_method must be equal to `average` or `max`' ) if ( word_pooling_method not in [ "average" , "max" , "concat" ] ) : raise ValueError ( 'word_pooling_method must be equal to `average`, `max` or `concat` ' ) if ( type ( batch_size ) != int ) : raise TypeError ( 'batch_size must be a positive integer' ) if ( batch_size <= 0 ) : raise ValueError ( 'batch_size must be a positive integer' ) if ( ( type ( path_to_save ) != str ) and ( path_to_save != None ) ) : raise TypeError ( 'path_to_save must be None or a string' ) if ( type ( layers ) != int ) : if ( type ( layers ) == list ) : layer_list = True for el in layers : if ( type ( el ) != int ) : raise TypeError ( 'layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer' ) elif ( el > self . nb_layer or el < 1 ) : raise ValueError ( 'layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer' ) else : raise TypeError ( 'layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer' ) else : if ( layers > self . nb_layer or layers < 1 ) : raise ValueError ( 'layers must be a int between 1 and nb_layer or a list of integers between 1 and nb_layer' ) texts_vectors = [ ] N = len ( input_ids_tensor ) counter = 0 with tqdm ( total = 100 ) as pbar : for b in self . __batch ( range ( 0 , N ) , batch_size ) : with torch . no_grad ( ) : encoded_layers_b = self . model ( input_ids_tensor [ b ] , masks_tensor [ b ] . to ( torch . int64 ) ) [ 1 ] if layer_list : for idx in b : if input_ids_tensor [ idx ] [ - 1 ] == 1 : eos_pos = 0 else : eos_pos = int ( ( input_ids_tensor [ idx ] == self . pad_id ) . nonzero ( ) [ 0 ] ) word_vector = self . __word_pooling ( encoded_layers_b , layers , idx - counter , word_pooling_method ) pooled_vector = self . __sentence_pooling ( word_vector [ : eos_pos - 1 ] [ 1 : ] , sentence_pooling_method ) texts_vectors . append ( pooled_vector ) else : words_vector = encoded_layers_b [ layers ] for idx in b : if input_ids_tensor [ idx ] [ - 1 ] == 1 : eos_pos = 0 else : eos_pos = int ( ( input_ids_tensor [ idx ] == self . pad_id ) . nonzero ( ) [ 0 ] ) pooled_vector = self . __sentence_pooling ( words_vector [ idx - counter ] [ : eos_pos - 1 ] [ 1 : ] , sentence_pooling_method ) texts_vectors . append ( pooled_vector ) counter += batch_size pbar . update ( np . round ( 100 * len ( b ) / N , 2 ) ) if path_to_save != None : torch . save ( texts_vectors , path_to_save + "text_vectors.pt" ) return texts_vectors
This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods.
This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods .
"""
            This function execute the forward pass of the input data into the BERT model and create a unique tensor for each input according to the stated pooling methods. 
            
            Parameters
            ----------
            input_ids_tensor : tensor
                Corresponds to then= ids of tokenized words. 
                Must match the output of the tokenize function.
            masks_tensor : tensor
                Corresponds to the attention masks of tokenized words. 
                Must match the output of the tokenize function. .
            sentence_pooling_method : str, optional
                Corresponds to the method of pooling to create a unique vector for each text
                The default is "average".
            word_pooling_method : str, optional
                Corresponds to the method of word pooling method in the case multiple layers have been stated. In other words it is the way to compute a single word vector from multiple layers.
                The default is "average".
            layers : int or list, optional
                Corresponds to the BERT layers to use to create the embeddings of words.
                The default is 11.
            batch_size : int, optional
                Size of batch when executing the forward pass into the BERT model, should get lower as your computational power gets lower. 
                The default is 50.
            path_to_save : str or None, optional
                Is the path to save the vector of texts if you want to avoid doing this computation again as it may be long. 
                if None, nothing will be saved. 
                The default is None.
            
            Returns
            -------
            texts_vectors : list
                A list of tensors, each tensor corresponding to an input text.

            """ #Just no to take into account BOS and EOS  #Just no to take into account BOS and EOS 
self
input_ids_tensor
masks_tensor
sentence_pooling_method
word_pooling_method
layers
batch_size
path_to_save