code: def getFileSize(filepath, shard):
      all_data = []
      for i in os.listdir(filepath):
          all_data.append(os.path.join(filepath, i))
      all_size = sum(
          [os.path.getsize(os.path.join(filepath, f)) for f in all_data]
      )
      ans = [
          [f.split("/")[-1], os.path.getsize(os.path.join(filepath, f))]
          for f in all_data
      ]
      ans = sorted(ans, key=lambda x: x[1], reverse=True)
      per_size = all_size / shard
      real_shard = []
      temp = []
      accu_size = 0
      for i in ans:
          accu_size += i[1]
          temp.append(i)
          if accu_size > per_size:
              real_shard.append(temp)
              accu_size = 0
              temp = []
  
      if len(temp) > 0:
          real_shard.append(temp)
  
      return real_shard
  
file_path: ColossalAI/examples/community/roberta/preprocessing/sentence_split.py
imports: import argparse
  import functools
  import json
  import multiprocessing
  import os
  import re
  import time
  from typing import List
  from tqdm import tqdm
label_desc: Use image node_id 3 for calling a global function with example usage: getFileSize(filepath, shard) and returns: real_shard