bin/bilm/data.py
def _load_shard(self, shard_name):
        """Read one file and convert to ids.

        Args:
            shard_name: file path.

        Returns:
            list of (id, char_id) tuples.
        """
        print('Loading data from: %s' % shard_name)
        with open(shard_name) as f:
            sentences_raw = f.readlines()
        if self._reverse:
            sentences = []
            for sentence in sentences_raw:
                splitted = sentence.split()
                splitted.reverse()
                sentences.append(' '.join(splitted))
        else:
            sentences = sentences_raw

        if self._shuffle_on_load:
            random.shuffle(sentences)
        ids = [self.vocab.encode(sentence, self._reverse) for sentence in sentences] #make sentence level
        if self._use_char_inputs:
            chars_ids = [self.vocab.encode_chars(sentence, self._reverse)
                     for sentence in sentences] # make sentence level
        else:
            chars_ids = [None] * len(ids)
        print('Loaded %d sentences.' % len(ids))
        print('Finished loading')
        return list(zip(ids, chars_ids))
Read one file and convert to ids.

        Args:
            shard_name: file path.

        Returns:
            list of (id, char_id) tuples.
        
Read one file and convert to ids.
Read one file and convert to ids .
def _load_shard(self, shard_name):
        print('Loading data from: %s' % shard_name)
        with open(shard_name) as f:
            sentences_raw = f.readlines()
        if self._reverse:
            sentences = []
            for sentence in sentences_raw:
                splitted = sentence.split()
                splitted.reverse()
                sentences.append(' '.join(splitted))
        else:
            sentences = sentences_raw
        if self._shuffle_on_load:
            random.shuffle(sentences)
        ids = [self.vocab.encode(sentence, self._reverse) for sentence in sentences] 
        if self._use_char_inputs:
            chars_ids = [self.vocab.encode_chars(sentence, self._reverse)
                     for sentence in sentences] 
        else:
            chars_ids = [None] * len(ids)
        print('Loaded %d sentences.' % len(ids))
        print('Finished loading')
        return list(zip(ids, chars_ids))
def _load_shard ( self , shard_name ) : print ( 'Loading data from: %s' % shard_name ) with open ( shard_name ) as f : sentences_raw = f . readlines ( ) if self . _reverse : sentences = [ ] for sentence in sentences_raw : splitted = sentence . split ( ) splitted . reverse ( ) sentences . append ( ' ' . join ( splitted ) ) else : sentences = sentences_raw if self . _shuffle_on_load : random . shuffle ( sentences ) ids = [ self . vocab . encode ( sentence , self . _reverse ) for sentence in sentences ] if self . _use_char_inputs : chars_ids = [ self . vocab . encode_chars ( sentence , self . _reverse ) for sentence in sentences ] else : chars_ids = [ None ] * len ( ids ) print ( 'Loaded %d sentences.' % len ( ids ) ) print ( 'Finished loading' ) return list ( zip ( ids , chars_ids ) )
Read one file and convert to ids.
Read one file and convert to ids .
"""Read one file and convert to ids.

        Args:
            shard_name: file path.

        Returns:
            list of (id, char_id) tuples.
        """ #make sentence level # make sentence level
list of (id, char_id) tuples.
list of ( id char_id ) tuples .
self
shard_name
None