bin/bilm/data.py
def encode(self, sentence, reverse=False, split=True):
        """Convert a sentence to a list of ids, with special tokens added.
        Sentence is a single string with tokens separated by whitespace.

        If reverse, then the sentence is assumed to be reversed, and
            this method will swap the BOS/EOS tokens appropriately."""

        if split:
            word_ids = [
                self.word_to_id(cur_word) for cur_word in sentence.split()
            ]
        else:
            word_ids = [self.word_to_id(cur_word) for cur_word in sentence]
        if reverse:
            #for i in range(current_sent_len, max_sentence_length):
            #    word_ids.append(self.bos)#pad ids
            return np.array([self.eos] + word_ids + [self.bos], dtype=np.int32)
        else:
            #for i in range(current_sent_len, max_sentence_length):
            #    word_ids.append(self.eos)#pad ids
            return np.array([self.bos] + word_ids + [self.eos], dtype=np.int32)
Convert a sentence to a list of ids, with special tokens added.
        Sentence is a single string with tokens separated by whitespace.

        If reverse, then the sentence is assumed to be reversed, and
            this method will swap the BOS/EOS tokens appropriately.
Convert a sentence to a list of ids, with special tokens added.
Sentence is a single string with tokens separated by whitespace.

If reverse, then the sentence is assumed to be reversed, and
this method will swap the BOS/EOS tokens appropriately.
Convert a sentence to a list of ids with special tokens added . Sentence is a single string with tokens separated by whitespace . If reverse then the sentence is assumed to be reversed and this method will swap the BOS / EOS tokens appropriately .
def encode(self, sentence, reverse=False, split=True):
        if split:
            word_ids = [
                self.word_to_id(cur_word) for cur_word in sentence.split()
            ]
        else:
            word_ids = [self.word_to_id(cur_word) for cur_word in sentence]
        if reverse:
            return np.array([self.eos] + word_ids + [self.bos], dtype=np.int32)
        else:
            return np.array([self.bos] + word_ids + [self.eos], dtype=np.int32)
def encode ( self , sentence , reverse = False , split = True ) : if split : word_ids = [ self . word_to_id ( cur_word ) for cur_word in sentence . split ( ) ] else : word_ids = [ self . word_to_id ( cur_word ) for cur_word in sentence ] if reverse : return np . array ( [ self . eos ] + word_ids + [ self . bos ] , dtype = np . int32 ) else : return np . array ( [ self . bos ] + word_ids + [ self . eos ] , dtype = np . int32 )
Convert a sentence to a list of ids, with special tokens added.
Convert a sentence to a list of ids with special tokens added .
"""Convert a sentence to a list of ids, with special tokens added.
        Sentence is a single string with tokens separated by whitespace.

        If reverse, then the sentence is assumed to be reversed, and
            this method will swap the BOS/EOS tokens appropriately.""" #for i in range(current_sent_len, max_sentence_length): #    word_ids.append(self.bos)#pad ids #for i in range(current_sent_len, max_sentence_length): #    word_ids.append(self.eos)#pad ids
self
sentence
reverse
split