code: def main():
      parser = argparse.ArgumentParser()
      parser.add_argument(
          "--source_tokenizer_dir",
          type=str,
          required=True,
          default=None,
          help="Source tokenizer directory",
      )
      parser.add_argument(
          "--target_tokenizer_dir",
          type=str,
          required=True,
          default=None,
          help="Target tokenizer directory",
      )
      parser.add_argument(
          "--expand_tokens_file",
          type=str,
          required=True,
          default=None,
          help="Path of the file containing tokens to be extended",
      )
      args = parser.parse_args()
  
      expand_tokens = []
      with open(
          file=args.expand_tokens_file, mode="r", encoding="utf-8"
      ) as fp_reader:
          for line in fp_reader:
              item = json.loads(line)
              # e.g., {"piece": "你好"}
              token = item["piece"]
              if token in expand_tokens:
                  continue
              expand_tokens.append(token)
      expand_tokens.sort(key=lambda t: len(t), reverse=False)
  
      expand_vocab_tokenizer(
          source_tokenizer_dir=args.source_tokenizer_dir,
          target_tokenizer_dir=args.target_tokenizer_dir,
          new_tokens=expand_tokens,
      )
  
file_path: ColossalAI/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
imports: import argparse
  import os
  import json
  from typing import List, Union
  from transformers.models.llama.tokenization_llama import LlamaTokenizer
  from sentencepiece import sentencepiece_model_pb2
  from colossalai.logging import get_dist_logger
label_desc: Use image node_id 2 for calling a global function with example usage: main() without return types