def METHOD_NAME(self, num_processes):
    if os.path.isdir(self.PATH):
        paths = [
            os.path.join(top, name) for top, _, names in os.walk(self.PATH)
            for name in names
        ]
        # paths = [entry.path for entry in os.scandir(self.PATH) if
        #          not entry.is_dir() and not entry.name.endswith("bz2")]
    else:
        paths = [self.PATH]
    task_queue, done_queue, info_queue = Queue(
        maxsize=self.TASK_QUEUE_LIMIT), Queue(
            maxsize=self.DONE_QUEUE_LIMIT), Queue()
    processes = []
    for i in range(num_processes):
        METHOD_NAME = Process(target=self.tokenize_worker,
                          args=(task_queue, done_queue, info_queue,
                                self.tokenizer, self.tokenize))
        METHOD_NAME.start()
        processes.append(METHOD_NAME)
    def read_input_to_queue():
        for path in paths:
            # if torch.cuda.is_available():print_rank_0(f"Start reading {path}")
            with open(path) as file:
                if self.split_row:
                    for row in file:
                        task_queue.put(row)
                else:
                    items = json.load(file)
                    for item in items["RECORDS"]:
                        task_queue.put(item)
        # if torch.cuda.is_available():print_rank_0("Read input complete")
        for i in range(len(processes)):
            task_queue.put('STOP')
    METHOD_NAME = Process(target=read_input_to_queue)
    METHOD_NAME.start()
    count = len(processes)
    progress_bar = tqdm.tqdm()
    while True:
        data = done_queue.get()
        if data == 'COMPLETE':
            count -= 1
            if count == 0:
                break
        else:
            self.write_result(data, self.writers)
            progress_bar.update()
    progress_bar.close()
    self.print_info(info_queue)