code: def get_parser():
      parser = argparse.ArgumentParser(
          formatter_class=argparse.RawDescriptionHelpFormatter,
          description=textwrap.dedent(
              r"""
      Learn BPE based on different implementations.
  
      We support the following models:
  
          - `nlp_process learn_subword --model spm --corpus CORPUS --vocab-size SIZE`
              Train a Sentencepiece Model on raw text.
  
          - `nlp_process learn_subword --model subword_nmt --corpus CORPUS --vocab-size SIZE`
              Train with the subword-nmt package:
  
          - `nlp_process learn_subword --model yttm --corpus CORPUS --vocab-size SIZE`
              Train with YouTokenToMe:
  
          - `nlp_process learn_subword --model hf_bytebpe --corpus CORPUS --vocab-size SIZE`
              Train with the Byte-level BPE Tokenizer Implemented by Huggingface.
  
          - `nlp_process learn_subword --model hf_wordpiece --corpus CORPUS --vocab-size SIZE`
              Train with the Wordpiece Tokenizer Implementated by Huggingface.
  
          - `nlp_process learn_subword --model hf_bpe --corpus CORPUS --vocab-size SIZE`
              Train with the BPE Tokenizer Implemented by Huggingface.
          
          - `nlp_process learn_subword --model spm --corpus CORPUS --vocab-size SIZE \
                                       --disable-bos --disable-eos \
                                       --custom-special-tokens "cls_token=[CLS]" "sep_token=[SEP]" "mask_token=[MASK]"`
              Train with the sentencepiece model and set cls token to "[CLS]", sep token to "[SEP]", and mask token to "[MASK]".
      """
          ),
          prog="learn_subword",
      )
      parser.add_argument(
          "--corpus",
          type=str,
          nargs="+",
          required=True,
          help="Path of the corpus. "
          "You may input multiple corpus files separated by space.",
      )
      parser.add_argument(
          "--vocab-size",
          type=int,
          required=True,
          help="Estimated learned vocabulary size",
      )
      parser.add_argument(
          "--model",
          type=str,
          choices=[
              "spm",
              "subword_nmt",
              "yttm",
              "hf_bytebpe",
              "hf_wordpiece",
              "hf_bpe",
          ],
          required=True,
          help="Subword model type",
      )
      parser.add_argument(
          "--save-dir",
          type=str,
          default=None,
          help="Directory for saving the model and vocabulary file",
      )
      parser.add_argument(
          "--coverage",
          type=float,
          default=1.0,
          help="Amount of characters covered by the model, "
          "this is only applicable to spm and yttm",
      )
      parser.add_argument(
          "--n-threads",
          type=int,
          default=-1,
          help="Number of threads, only applicable to yttm",
      )
      parser.add_argument(
          "--input-sentence-size",
          type=int,
          default=1000000,
          help="Size of input sentence. Internally, the algorithm will randomly "
          "sample some sentences to train the tokenizer. "
          "This is only applicable to sentencepiece, "
          "you can reduce this value when getting out of memory error.",
      )
      parser.add_argument(
          "--lowercase",
          action="store_true",
          default=False,
          help="Use lowercase, "
          "only applicable to hf_bpe, hf_bytebpe and hf_wordpiece",
      )
      parser.add_argument(
          "--bert-normalizer",
          action="store_true",
          default=False,
          help="Whether to use the Normalizer in BERT. "
          "This will be used only when you choose the huggingface models. "
          "Basically, the BERT Normalizer will "
          "1) remove control characters. "
          "2) putting spaces around chinese characters. "
          "3) strip accents. "
          "For more details, you can refer to "
          "https://github.com/google-research/bert/"
          "blob/master/multilingual.md#tokenization",
      )
      parser.add_argument(
          "--split-punctuation",
          action="store_true",
          default=False,
          help="Whether to split on punctuation.",
      )
      parser.add_argument(
          "--disable-unk",
          action="store_true",
          default=False,
          help="Whether to disable unk token (default settings enable unk)",
      )
      parser.add_argument(
          "--disable-bos",
          action="store_true",
          default=False,
          help="Disable bos token (default settings enable bos)",
      )
      parser.add_argument(
          "--disable-eos",
          action="store_true",
          default=False,
          help="Disable eos token (default settings enable eos)",
      )
      parser.add_argument(
          "--disable-pad",
          action="store_true",
          default=False,
          help="Disable pad token (default settings enable pad)",
      )
      parser.add_argument(
          "--custom-special-tokens",
          type=str,
          nargs="*",
          default=[],
          help="Specified special tokens key value pairs besides unk, "
          "bos, eos and pad, for example: "
          '--custom-special-tokens "cls_token=<cls>" "sep_token=<sep>", '
          "this is not applicable to yttm",
      )
      return parser
  
file_path: gluon-nlp/scripts/processing/learn_subword.py
imports: from gluonnlp.utils.lazy_imports import try_import_sentencepiece, try_import_subword_nmt, try_import_yttm, try_import_huggingface_tokenizers
  from pkg_resources import parse_version
  import argparse
  import warnings
  import textwrap
  import os
  from collections import OrderedDict
  import json
  from uuid import uuid4
  from packaging import version
  from gluonnlp.data import Vocab
label_desc: Use image node_id 1 for calling a global function with example usage: get_parser() and returns: parser