class_docstr: Process a pair of corpus.
  
  This largely recovers the functionality of 'clean-corpus-n.perl' in mosesdecoder.
  The difference is that it is customizable with pure python.
  
  By default, we will perform the following pre-processing pipeline.
  Each stage could be turned on/off and specialized based on the input arguments.
  Also, you may directly revise the code and write your own processing script.
  
  1. Normalize sentence
  2. Pre-filter
  3. Tokenize the sentence
  4. Filter the sentence based on different rules
      3.1 Remove pairs where `max(len(lhs) / len(rhs), len(rhs) / len(lhs) > max_ratio`
      3.2 Remove pairs where not `min_max_words <= len(lhs) <= max_num_words` and
                                 `min_max_words <= len(rhs) <= max_num_words`
class_docstr_tok: ["Process","a","pair","of","corpus",".","This","largely","recovers","the","functionality","of","'clean-corpus-n.perl","'","in","mosesdecoder",".","The","difference","is","that","it","is","customizable","with","pure","python",".","By","default",",","we","will","perform","the","following","pre-processing","pipeline",".","Each","stage","could","be","turned","on\/off","and","specialized","based","on","the","input","arguments",".","Also",",","you","may","directly","revise","the","code","and","write","your","own","processing","script",".","1",".","Normalize","sentence","2",".","Pre-filter","3",".","Tokenize","the","sentence","4",".","Filter","the","sentence","based","on","different","rules","3.1","Remove","pairs","where","`","max","(","len","(","lhs",")","\/","len","(","rhs",")",",","len","(","rhs",")","\/","len","(","lhs",")",">","max_ratio","`","3.2","Remove","pairs","where","not","`","min_max_words","<","=","len","(","lhs",")","<","=","max_num_words","`","and","`","min_max_words","<","=","len","(","rhs",")","<","=","max_num_words","`"]
code: def __init__(
      self,
      src_lang: str,
      tgt_lang: str,
      normalize: bool = True,
      src_tokenizer: Union[str, BaseTokenizer] = "whitespace",
      tgt_tokenizer: Union[str, BaseTokenizer] = "whitespace",
      max_ratio: Optional[float] = None,
      min_num_words: Optional[int] = None,
      max_num_words: Optional[int] = None,
      discard_non_latin1: bool = False,
  ):
      self._src_lang = src_lang
      self._tgt_lang = tgt_lang
      if normalize:
          self._src_normalizer = MosesNormalizer(lang=src_lang)
          self._tgt_normalizer = MosesNormalizer(lang=tgt_lang)
      self._src_tokenizer = get_tokenizer(src_tokenizer, src_lang)
      self._tgt_tokenizer = get_tokenizer(tgt_tokenizer, tgt_lang)
      self._max_ratio = max_ratio
      self._min_num_words = min_num_words
      self._max_num_words = max_num_words
      self._discard_non_latin1 = discard_non_latin1
  
imports: import argparse
  import os
  import multiprocessing
  import time
  import numpy
  import warnings
  import re
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data.tokenizers import MosesTokenizer, BaseTokenizer, WhitespaceTokenizer, JiebaTokenizer
  from typing import List, Union, Optional
label_desc: Use image node_id 1 to create a new ParallelCorpusProcessor object  with example: obj = ParallelCorpusProcessor(src_lang, tgt_lang, normalize, src_tokenizer, tgt_tokenizer, max_ratio, min_num_words, max_num_words, discard_non_latin1)