code: def get_file_stats(file_path, tqdm_func, global_tqdm):
      reader = Reader()
      total_documents = 0
      total_size = 0
      update_frequency = 10000
      current_file_position = 0
  
      with tqdm_func(
          total=os.path.getsize(file_path),
          dynamic_ncols=True,
          unit="byte",
          unit_scale=1,
      ) as progress:
          for document in reader.read(file_path, get_meta=True):
              total_size += len(document)
              total_documents += 1
  
              if total_documents % update_frequency == 0:
                  new_file_pos = reader.fh.tell()
                  bytes_read = new_file_pos - current_file_position
                  current_file_position = new_file_pos
                  progress.update(bytes_read)
                  global_tqdm.update(bytes_read)
  
      return (total_documents, total_size)
  
file_path: lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py
imports: from lm_eval.decontamination.archiver import Reader
  import os
  import json
  from functools import reduce
  import glob
  import tqdm
  from tqdm_multiprocess import TqdmMultiProcessPool
label_desc: Use image node_id 1 for calling a global function with example usage: get_file_stats(file_path, tqdm_func, global_tqdm) and returns: total_documents,  total_size