class_docstr: Process a pair of corpus.
  
  This largely recovers the functionality of 'clean-corpus-n.perl' in mosesdecoder.
  The difference is that it is customizable with pure python.
  
  By default, we will perform the following pre-processing pipeline.
  Each stage could be turned on/off and specialized based on the input arguments.
  Also, you may directly revise the code and write your own processing script.
  
  1. Normalize sentence
  2. Pre-filter
  3. Tokenize the sentence
  4. Filter the sentence based on different rules
      3.1 Remove pairs where `max(len(lhs) / len(rhs), len(rhs) / len(lhs) > max_ratio`
      3.2 Remove pairs where not `min_max_words <= len(lhs) <= max_num_words` and
                                 `min_max_words <= len(rhs) <= max_num_words`
class_docstr_tok: ["Process","a","pair","of","corpus",".","This","largely","recovers","the","functionality","of","'clean-corpus-n.perl","'","in","mosesdecoder",".","The","difference","is","that","it","is","customizable","with","pure","python",".","By","default",",","we","will","perform","the","following","pre-processing","pipeline",".","Each","stage","could","be","turned","on\/off","and","specialized","based","on","the","input","arguments",".","Also",",","you","may","directly","revise","the","code","and","write","your","own","processing","script",".","1",".","Normalize","sentence","2",".","Pre-filter","3",".","Tokenize","the","sentence","4",".","Filter","the","sentence","based","on","different","rules","3.1","Remove","pairs","where","`","max","(","len","(","lhs",")","\/","len","(","rhs",")",",","len","(","rhs",")","\/","len","(","lhs",")",">","max_ratio","`","3.2","Remove","pairs","where","not","`","min_max_words","<","=","len","(","lhs",")","<","=","max_num_words","`","and","`","min_max_words","<","=","len","(","rhs",")","<","=","max_num_words","`"]
code: def process_chunk(self, args):
      (
          src_path,
          src_chunk_start,
          src_chunk_size,
          tgt_path,
          tgt_chunk_start,
          tgt_chunk_size,
      ) = args
      processed_src_lines = []
      processed_tgt_lines = []
      with open(src_path, "rb") as src_in_f:
          with open(tgt_path, "rb") as tgt_in_f:
              # Read chunk from source and target
              src_in_f.seek(src_chunk_start)
              src_lines = src_in_f.read(src_chunk_size)
              tgt_in_f.seek(tgt_chunk_start)
              tgt_lines = tgt_in_f.read(tgt_chunk_size)
              src_lines = src_lines.splitlines()
              tgt_lines = tgt_lines.splitlines()
              unfiltered_line_num = len(src_lines)
              for src_line, tgt_line in zip(src_lines, tgt_lines):
                  src_line = src_line.decode("utf-8").strip()
                  tgt_line = tgt_line.decode("utf-8").strip()
                  # 1. Normalize
                  src_line = self._src_normalizer(src_line)
                  tgt_line = self._tgt_normalizer(tgt_line)
                  # 2. Filter after normalization.
                  if self._discard_non_latin1:
                      if not check_both_latin1(src_line, tgt_line):
                          continue
                  # 3. Tokenize the sentence
                  src_tokens = self._src_tokenizer.encode(src_line)
                  tgt_tokens = self._tgt_tokenizer.encode(tgt_line)
                  # 4. Filter after tokenization. Filter with multiple rules
                  if len(src_tokens) == 0 or len(tgt_tokens) == 0:
                      continue
                  if self._max_ratio is not None:
                      if (
                          max(
                              len(src_tokens) / len(tgt_tokens),
                              len(tgt_tokens) / len(src_tokens),
                          )
                          > self._max_ratio
                      ):
                          continue
                  if self._max_num_words is not None:
                      if (
                          len(src_tokens) > self._max_num_words
                          or len(tgt_tokens) > self._max_num_words
                      ):
                          continue
                  if self._min_num_words is not None:
                      if (
                          len(src_tokens) < self._min_num_words
                          or len(tgt_tokens) < self._min_num_words
                      ):
                          continue
                  processed_src_lines.append(" ".join(src_tokens))
                  processed_tgt_lines.append(" ".join(tgt_tokens))
      return (
          processed_src_lines,
          processed_tgt_lines,
          unfiltered_line_num,
      )
  
imports: import argparse
  import os
  import multiprocessing
  import time
  import numpy
  import warnings
  import re
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data.tokenizers import MosesTokenizer, BaseTokenizer, WhitespaceTokenizer, JiebaTokenizer
  from typing import List, Union, Optional
label_desc: Use image node_id 2 for calling the ParallelCorpusProcessor obj's underlying member method code with example usage: obj.process_chunk(args) and returns: processed_src_lines,  processed_tgt_lines,  unfiltered_line_num