code: def get_stats():
      files = get_files()
      total_size_bytes = sum(map(lambda x: os.path.getsize(x), files))
  
      pool = TqdmMultiProcessPool(4)
      global_tqdm = tqdm.tqdm(
          total=total_size_bytes,
          dynamic_ncols=True,
          unit="byte",
          unit_scale=1,
      )
  
      # Generate minhashes with pool
      tasks = [(get_file_stats, (file,)) for file in files]
  
      def on_done(_):
          return None
  
      def on_error(_):
          return None
  
      results = pool.map(global_tqdm, tasks, on_error, on_done)
  
      total_documents, total_size = reduce(
          lambda x, y: (x[0] + y[0], x[1] + y[1]), results
      )
  
      start_offsets = []
      current_offset = 0
      for file_document_count, _ in results:
          start_offsets.append(current_offset)
          current_offset += file_document_count
  
      return (total_documents, total_size, start_offsets)
  
file_path: lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py
imports: from lm_eval.decontamination.archiver import Reader
  import os
  import json
  from functools import reduce
  import glob
  import tqdm
  from tqdm_multiprocess import TqdmMultiProcessPool
label_desc: Use image node_id 3 for calling a global function with example usage: get_stats() and returns: total_documents,  total_size,  start_offsets, None, None