code: def sharding(config, out_file):
      with open(out_file, "rb") as fr:
          captions = pickle.load(fr)
      target_dir = config.target_dir
      prefix = (
          os.path.basename(os.path.splitext(config.caption_pkl_path)[0])
          + "."
          + config.bert_name
          + "."
      )
      for split in ["train", "val"]:
          target_path = os.path.join(target_dir, split + "_meta")
          with open(target_path + ".pkl", "rb") as fr:
              meta = pickle.load(fr)
          print("load meta", target_path, len(meta))
          for shard_id in meta:
              numpify(
                  shard_id,
                  meta[shard_id],
                  captions,
                  target_dir,
                  split,
                  prefix,
              )
  
file_path: fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py
imports: import pickle
  import os
  import argparse
  import numpy
  from torch.utils.data import Dataset, DataLoader
  from mmpt.processors import PKLJSONStrTextProcessor
  from mmpt.utils import ShardedTensor, recursive_config
label_desc: Use image node_id 2 for calling a global function with example usage: sharding(config, out_file) without return types