bin/bilm/data.py
def _load_random_shard(self):
        """Randomly select a file and read it."""
        if self._test:
            if len(self._all_shards) == 0:
                # we've loaded all the data
                # this will propogate up to the generator in get_batch
                # and stop iterating
                raise StopIteration
            else:
                shard_name = self._all_shards.pop()
        else:
            # just pick a random shard
            shard_name = self._choose_random_shard()

        ids = self._load_shard(shard_name)
        self._i = 0
        self._nids = len(ids)
        return ids
Randomly select a file and read it.
Randomly select a file and read it.
Randomly select a file and read it .
def _load_random_shard(self):
        if self._test:
            if len(self._all_shards) == 0:
                raise StopIteration
            else:
                shard_name = self._all_shards.pop()
        else:
            shard_name = self._choose_random_shard()
        ids = self._load_shard(shard_name)
        self._i = 0
        self._nids = len(ids)
        return ids
def _load_random_shard ( self ) : if self . _test : if len ( self . _all_shards ) == 0 : raise StopIteration else : shard_name = self . _all_shards . pop ( ) else : shard_name = self . _choose_random_shard ( ) ids = self . _load_shard ( shard_name ) self . _i = 0 self . _nids = len ( ids ) return ids
Randomly select a file and read it.
Randomly select a file and read it .
"""Randomly select a file and read it.""" # we've loaded all the data # this will propogate up to the generator in get_batch # and stop iterating # just pick a random shard
self