code: def get_tokenizer(args):
      tokenizer = get_nmt_tokenizer(
          library=args.tokenizer_library,
          model_name=args.tokenizer_type,
          tokenizer_model=args.tokenizer_model,
          vocab_file=args.vocab_file,
          merges_file=args.merge_file,
          delimiter=args.delimiter,
      )
      if args.need_pad_id:
          if not hasattr(tokenizer, "pad_id"):
              tokenizer.add_special_tokens({"pad_token": "<pad>"})
          elif hasattr(tokenizer, "pad_id") and (
              tokenizer.pad_id is None or tokenizer.pad_id < 0
          ):
              tokenizer.add_special_tokens({"pad_token": "<pad>"})
      return tokenizer
  
imports: import argparse
  import gzip
  import json
  import multiprocessing
  import os
  import pathlib
  import sys
  import time
  import ftfy
  import torch
  from nemo.collections.nlp.data.language_modeling.megatron import indexed_dataset
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer
label_desc: Use image node_id 1 for calling a global function with example usage: get_tokenizer(args) and returns: tokenizer