code: def get_sent(
      output_path, input_path, fin_list=[], host=-1, seq_len=512
  ) -> None:
      workers = 32
  
      if input_path[-1] == "/":
          input_path = input_path[:-1]
  
      cur_path = os.path.join(output_path, str(host) + ".txt")
      new_split_sentence = functools.partial(
          split_sentence, limit=seq_len - 2
      )
      with open(cur_path, "w", encoding="utf-8") as f:
          for fi, fin_path in enumerate(fin_list):
              if not os.path.exists(
                  os.path.join(input_path, fin_path[0])
              ):
                  continue
              if ".json" not in fin_path[0]:
                  continue
  
              print("Processing ", fin_path[0], " ", fi)
  
              with open(
                  os.path.join(input_path, fin_path[0]), "r"
              ) as fin:
                  f_data = [l["content"] for l in json.load(fin)]
  
                  pool = multiprocessing.Pool(workers)
                  all_sent = pool.imap_unordered(
                      new_split_sentence, f_data, 32
                  )
                  pool.close()
              print("finished..")
  
              cnt = 0
              for d in tqdm(all_sent):
                  for i in d:
                      f.write(i.strip() + "\n")
                  f.write("]]" + "\n")
                  cnt += 1
  
file_path: ColossalAI/examples/community/roberta/preprocessing/sentence_split.py
imports: import argparse
  import functools
  import json
  import multiprocessing
  import os
  import re
  import time
  from typing import List
  from tqdm import tqdm
label_desc: Use image node_id 2 for calling a global function with example usage: get_sent(output_path, input_path, fin_list, host, seq_len) without return types