bin/bilm/data.py
def batch_sentences(self, sentences: List[List[str]]):
        '''
        Batch the sentences as character ids
        Each sentence is a list of tokens without <s> or </s>, e.g.
        [['The', 'first', 'sentence', '.'], ['Second', '.']]
        '''
        n_sentences = len(sentences)
        max_length = max(len(sentence) for sentence in sentences) + 2

        X_ids = np.zeros((n_sentences, max_length), dtype=np.int64)

        for k, sent in enumerate(sentences):
            length = len(sent) + 2
            ids_without_mask = self._lm_vocab.encode(sent, split=False)
            # add one so that 0 is the mask value
            X_ids[k, :length] = ids_without_mask + 1

        return X_ids

        Batch the sentences as character ids
        Each sentence is a list of tokens without <s> or </s>, e.g.
        [['The', 'first', 'sentence', '.'], ['Second', '.']]
        
Batch the sentences as character ids
Each sentence is a list of tokens without  or , e.g.
Batch the sentences as character ids Each sentence is a list of tokens without or e . g .
def batch_sentences(self, sentences: List[List[str]]):
        n_sentences = len(sentences)
        max_length = max(len(sentence) for sentence in sentences) + 2
        X_ids = np.zeros((n_sentences, max_length), dtype=np.int64)
        for k, sent in enumerate(sentences):
            length = len(sent) + 2
            ids_without_mask = self._lm_vocab.encode(sent, split=False)
            X_ids[k, :length] = ids_without_mask + 1
        return X_ids
def batch_sentences ( self , sentences : List [ List [ str ] ] ) : n_sentences = len ( sentences ) max_length = max ( len ( sentence ) for sentence in sentences ) + 2 X_ids = np . zeros ( ( n_sentences , max_length ) , dtype = np . int64 ) for k , sent in enumerate ( sentences ) : length = len ( sent ) + 2 ids_without_mask = self . _lm_vocab . encode ( sent , split = False ) X_ids [ k , : length ] = ids_without_mask + 1 return X_ids
Batch the sentences as character ids
Each sentence is a list of tokens without <s> or </s>, e.g.
Batch the sentences as character ids Each sentence is a list of tokens without <s > or < / s > e . g .
'''
        Batch the sentences as character ids
        Each sentence is a list of tokens without <s> or </s>, e.g.
        [['The', 'first', 'sentence', '.'], ['Second', '.']]
        ''' # add one so that 0 is the mask value
self
sentences
List[List[str]]