code: def multiprocess_reader(readers, use_pipe=True, queue_size=1000):
      """
      multiprocess_reader use python multi process to read data from readers
      and then use multiprocess.Queue or multiprocess.Pipe to merge all
      data. The process number is equal to the number of input readers, each
      process call one reader.
      Multiprocess.Queue require the rw access right to /dev/shm, some
      platform does not support.
      you need to create multiple readers first, these readers should be independent
      to each other so that each process can work independently.
      An example:
      .. code-block:: python
          reader0 = reader(["file01", "file02"])
          reader1 = reader(["file11", "file12"])
          reader1 = reader(["file21", "file22"])
          reader = multiprocess_reader([reader0, reader1, reader2],
              queue_size=100, use_pipe=False)
      """
  
      assert type(readers) is list and len(readers) > 0
  
      def _read_into_queue(reader, queue):
          """read_into_queue"""
          for sample in reader():
              if sample is None:
                  raise ValueError("sample has None")
              queue.put(serialize_data(sample))
          queue.put(serialize_data(None))
  
      def queue_reader():
          """queue_reader"""
          queue = multiprocessing.Queue(queue_size)
          for reader in readers:
              p = multiprocessing.Process(
                  target=_read_into_queue, args=(reader, queue)
              )
              p.start()
  
          reader_num = len(readers)
          finish_num = 0
          while finish_num < reader_num:
              sample = deserialize_data(queue.get())
              if sample is None:
                  finish_num += 1
              else:
                  yield sample
  
      def _read_into_pipe(reader, conn):
          """read_into_pipe"""
          for sample in reader():
              if sample is None:
                  raise ValueError("sample has None!")
              conn.send(serialize_data(sample))
          conn.send(serialize_data(None))
          conn.close()
  
      def pipe_reader():
          """pipe_reader"""
          conns = []
          for reader in readers:
              parent_conn, child_conn = multiprocessing.Pipe()
              conns.append(parent_conn)
              p = multiprocessing.Process(
                  target=_read_into_pipe, args=(reader, child_conn)
              )
              p.start()
  
          reader_num = len(readers)
          finish_num = 0
          conn_to_remove = []
          finish_flag = np.zeros(len(conns), dtype="int32")
          while finish_num < reader_num:
              for conn_id, conn in enumerate(conns):
                  if finish_flag[conn_id] > 0:
                      continue
                  buff = conn.recv()
                  now = time.time()
                  sample = deserialize_data(buff)
                  out = time.time() - now
                  if sample is None:
                      finish_num += 1
                      conn.close()
                      finish_flag[conn_id] = 1
                  else:
                      yield sample
  
      if use_pipe:
          return pipe_reader
      else:
          return queue_reader
  
file_path: PGL/legacy/examples/distribute_metapath2vec/mp_reader.py
imports: import multiprocessing
  import numpy
  import time
  import paddle.fluid
  import pyarrow
label_desc: Use image node_id 5 for calling a global function with example usage: multiprocess_reader(readers, use_pipe, queue_size) and returns: pipe_reader, queue_reader