code: def expand_vocab_tokenizer(
      source_tokenizer_dir: Union[str, os.PathLike],
      target_tokenizer_dir: Union[str, os.PathLike],
      new_tokens: List[str],
  ) -> None:
      """Expand tokenizer for continue pre-training."""
      if os.path.exists(target_tokenizer_dir):
          raise RuntimeError(
              f"Find existed directory {target_tokenizer_dir}"
          )
  
      source_tokenizer = LlamaTokenizer.from_pretrained(
          source_tokenizer_dir
      )
      logger.info(source_tokenizer)
      source_sp_processor = source_tokenizer.sp_model
      source_spm = sp_pb2_model.ModelProto()
      source_spm.ParseFromString(
          source_sp_processor.serialized_model_proto()
      )
  
      logger.info(f"Source tokenizer size: {len(source_sp_processor)}")
  
      # Add new tokens to source tokenizer.
      source_spm_tokens = set([p.piece for p in source_spm.pieces])
      for piece in new_tokens:
          assert isinstance(
              piece, str
          ), f"Invalid token({piece}) type {type(piece)}"
          if piece in source_spm_tokens:
              # Skip existed token.
              continue
          new_p = sp_pb2_model.ModelProto().SentencePiece()
          new_p.piece = piece
          new_p.score = 0
          source_spm.pieces.append(new_p)
      logger.info(
          f"Expand vocab from {len(source_spm_tokens)} to {len(source_spm.pieces)}"
      )
  
      # Save
      os.makedirs(target_tokenizer_dir)
      target_tokenizer_model_path = os.path.join(
          target_tokenizer_dir, "tokenizer.model"
      )
      with open(file=target_tokenizer_model_path, mode="wb") as fp:
          fp.write(source_spm.SerializeToString())
  
      target_tokenizer = LlamaTokenizer(
          vocab_file=target_tokenizer_model_path
      )
      target_tokenizer.save_pretrained(
          save_directory=target_tokenizer_dir
      )
      logger.info(
          f"Successfully save expand tokenizer to {target_tokenizer_dir}"
      )
  
file_path: colossalai/applications/Colossal-LLaMA-2/colossal_llama2/tokenizer/init_tokenizer.py
imports: import argparse
  import os
  import json
  from typing import List, Union
  from transformers.models.llama.tokenization_llama import LlamaTokenizer
  from sentencepiece import sentencepiece_model_pb2
  from colossalai.logging import get_dist_logger
label_desc: Use image node_id 1 for calling a global function with example usage: expand_vocab_tokenizer(source_tokenizer_dir, target_tokenizer_dir, new_tokens) without return types