examples/language_model/experimental/ernie/propeller/data/functional.py
def _buffered_func(dataset, size):
    """
    Creates a buffered data reader.

    The buffered data reader will read and save data entries into a
    buffer. Reading from the buffered data reader will proceed as long
    as the buffer is not empty.

    :param reader: the data reader to read from.
    :type reader: callable
    :param size: max buffer size.
    :type size: int

    :returns: the buffered data reader.
    """

    class _EndSignal(object):
        pass

    end = _EndSignal()

    def _read_worker(r, q):
        for d in r:
            q.put(d)
        q.put(end)

    def _data_reader():
        r = dataset()
        q = multiprocessing.Queue(maxsize=size)
        t = multiprocessing.Process(
            target=_read_worker, args=(
                r,
                q, ))
        t.daemon = True
        t.start()
        e = q.get()
        while e != end:
            yield e
            e = q.get()

    return _data_reader

    Creates a buffered data reader.

    The buffered data reader will read and save data entries into a
    buffer. Reading from the buffered data reader will proceed as long
    as the buffer is not empty.

    :param reader: the data reader to read from.
    :type reader: callable
    :param size: max buffer size.
    :type size: int

    :returns: the buffered data reader.
    
Creates a buffered data reader.
The buffered data reader will read and save data entries into a
buffer. Reading from the buffered data reader will proceed as long
as the buffer is not empty.
Creates a buffered data reader . The buffered data reader will read and save data entries into a buffer . Reading from the buffered data reader will proceed as long as the buffer is not empty .
def _buffered_func(dataset, size):
    class _EndSignal(object):
        pass
    end = _EndSignal()
    def _read_worker(r, q):
        for d in r:
            q.put(d)
        q.put(end)
    def _data_reader():
        r = dataset()
        q = multiprocessing.Queue(maxsize=size)
        t = multiprocessing.Process(
            target=_read_worker, args=(
                r,
                q, ))
        t.daemon = True
        t.start()
        e = q.get()
        while e != end:
            yield e
            e = q.get()
    return _data_reader
def _buffered_func ( dataset , size ) : class _EndSignal ( object ) : pass end = _EndSignal ( ) def _read_worker ( r , q ) : for d in r : q . put ( d ) q . put ( end ) def _data_reader ( ) : r = dataset ( ) q = multiprocessing . Queue ( maxsize = size ) t = multiprocessing . Process ( target = _read_worker , args = ( r , q , ) ) t . daemon = True t . start ( ) e = q . get ( ) while e != end : yield e e = q . get ( ) return _data_reader
Creates a buffered data reader.
Creates a buffered data reader .
"""
    Creates a buffered data reader.

    The buffered data reader will read and save data entries into a
    buffer. Reading from the buffered data reader will proceed as long
    as the buffer is not empty.

    :param reader: the data reader to read from.
    :type reader: callable
    :param size: max buffer size.
    :type size: int

    :returns: the buffered data reader.
    """
the buffered data reader.
the buffered data reader .
dataset
size
max buffer size.
max buffer size .
reader
the data reader to read from.
the data reader to read from .