class_docstr: Process sentence of corpus.
  
  This largely recovers the functionality of 'clean-corpus-n.perl' in mosesdecoder.
  The difference is that it is customizable with pure python.
  
  By default, we will perform the following pre-processing pipeline.
  Each stage could be turned on/off and specialized based on the input arguments.
  Also, you may directly revise the code and write your own processing script.
  
  1. Normalize sentence
  2. Pre-filter
  3. Tokenize the sentence
  4. Filter the sentence based on different rules
      3.1 Remove sentences where `max(len(lhs) / len(rhs), len(rhs) / len(lhs) > max_ratio`
      3.2 Remove sentences where not `min_max_words <= len(lhs) <= max_num_words` and
                                     `min_max_words <= len(rhs) <= max_num_words`
class_docstr_tok: ["Process","sentence","of","corpus",".","This","largely","recovers","the","functionality","of","'clean-corpus-n.perl","'","in","mosesdecoder",".","The","difference","is","that","it","is","customizable","with","pure","python",".","By","default",",","we","will","perform","the","following","pre-processing","pipeline",".","Each","stage","could","be","turned","on\/off","and","specialized","based","on","the","input","arguments",".","Also",",","you","may","directly","revise","the","code","and","write","your","own","processing","script",".","1",".","Normalize","sentence","2",".","Pre-filter","3",".","Tokenize","the","sentence","4",".","Filter","the","sentence","based","on","different","rules","3.1","Remove","sentences","where","`","max","(","len","(","lhs",")","\/","len","(","rhs",")",",","len","(","rhs",")","\/","len","(","lhs",")",">","max_ratio","`","3.2","Remove","sentences","where","not","`","min_max_words","<","=","len","(","lhs",")","<","=","max_num_words","`","and","`","min_max_words","<","=","len","(","rhs",")","<","=","max_num_words","`"]
code: def process_chunk(self, args):
      path, chunk_start, chunk_size = args
      processed_lines = []
      with open(path, "rb") as in_f:
          # Read chunk
          in_f.seek(chunk_start)
          lines = in_f.read(chunk_size)
          lines = lines.splitlines()
          unfiltered_line_num = len(lines)
          for line in lines:
              line = line.decode("utf-8").strip()
              # 1. Normalize
              line = self._normalizer(line)
              # 2. Filter after normalization.
              if self._discard_non_latin1:
                  if not check_latin1(line):
                      continue
              # 3. Tokenize the sentence
              tokens = self._tokenizer.encode(line)
              # 4. Filter after tokenization. Filter with multiple rules
              if len(tokens) == 0:
                  continue
              if self._max_num_words is not None:
                  if len(tokens) > self._max_num_words:
                      continue
              if self._min_num_words is not None:
                  if len(tokens) < self._min_num_words:
                      continue
              processed_lines.append(" ".join(tokens))
      return processed_lines, unfiltered_line_num
  
imports: import argparse
  import os
  import multiprocessing
  import time
  import numpy
  import warnings
  import re
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data.tokenizers import MosesTokenizer, BaseTokenizer, WhitespaceTokenizer, JiebaTokenizer
  from typing import List, Union, Optional
label_desc: Use image node_id 2 for calling the MonoCorpusProcessor obj's underlying member method code with example usage: obj.process_chunk(args) and returns: processed_lines,  unfiltered_line_num