class_docstr: Process sentence of corpus.
  
  This largely recovers the functionality of 'clean-corpus-n.perl' in mosesdecoder.
  The difference is that it is customizable with pure python.
  
  By default, we will perform the following pre-processing pipeline.
  Each stage could be turned on/off and specialized based on the input arguments.
  Also, you may directly revise the code and write your own processing script.
  
  1. Normalize sentence
  2. Pre-filter
  3. Tokenize the sentence
  4. Filter the sentence based on different rules
      3.1 Remove sentences where `max(len(lhs) / len(rhs), len(rhs) / len(lhs) > max_ratio`
      3.2 Remove sentences where not `min_max_words <= len(lhs) <= max_num_words` and
                                     `min_max_words <= len(rhs) <= max_num_words`
class_docstr_tok: ["Process","sentence","of","corpus",".","This","largely","recovers","the","functionality","of","'clean-corpus-n.perl","'","in","mosesdecoder",".","The","difference","is","that","it","is","customizable","with","pure","python",".","By","default",",","we","will","perform","the","following","pre-processing","pipeline",".","Each","stage","could","be","turned","on\/off","and","specialized","based","on","the","input","arguments",".","Also",",","you","may","directly","revise","the","code","and","write","your","own","processing","script",".","1",".","Normalize","sentence","2",".","Pre-filter","3",".","Tokenize","the","sentence","4",".","Filter","the","sentence","based","on","different","rules","3.1","Remove","sentences","where","`","max","(","len","(","lhs",")","\/","len","(","rhs",")",",","len","(","rhs",")","\/","len","(","lhs",")",">","max_ratio","`","3.2","Remove","sentences","where","not","`","min_max_words","<","=","len","(","lhs",")","<","=","max_num_words","`","and","`","min_max_words","<","=","len","(","rhs",")","<","=","max_num_words","`"]
docstr: Preprocess the mono corpus
  
  Parameters
  ----------
  corpus_paths
      Corpus paths
  out_path
      Write the results to the output path
  chunk_size
      Approximately split the corpus files into multiple chunks
  num_process
      The number of process
  
  Returns
  -------
  line_count
      The number of lines in the final filtered file
docstr_tok: ["Preprocess","the","mono","corpus","Parameters","--","--","--","--","--","corpus_paths","Corpus","paths","out_path","Write","the","results","to","the","output","path","chunk_size","Approximately","split","the","corpus","files","into","multiple","chunks","num_process","The","number","of","process","Returns","--","--","--","-","line_count","The","number","of","lines","in","the","final","filtered","file"]
code: def process_mono_corpus(
      self,
      corpus_paths: List[str],
      out_path: str,
      chunk_size: int = 1024 * 1024,
      num_process: int = 8,
  ) -> int:
      """Preprocess the mono corpus
  
      Parameters
      ----------
      corpus_paths
          Corpus paths
      out_path
          Write the results to the output path
      chunk_size
          Approximately split the corpus files into multiple chunks
      num_process
          The number of process
  
      Returns
      -------
      line_count
          The number of lines in the final filtered file
      """
      start = time.time()
      total_line_count = 0
      filtered_line_count = 0
  
      def chunk_iterator(step=10):
          for path in corpus_paths:
              line_pos = get_line_byte_start(path)
              line_size = line_pos[1:] - line_pos[:-1]
              num_lines = line_pos.shape[0] - 1
              budget = chunk_size
              chunk_start = 0
              cur_chunk_size = 0
              for i in range(0, num_lines, step):
                  line_batch_num = min(num_lines - i, step)
                  batch_line_size = line_size[
                      i : (i + line_batch_num)
                  ].sum()
                  budget -= batch_line_size
                  cur_chunk_size += batch_line_size
                  if budget <= 0 or i + step >= num_lines:
                      yield path, chunk_start, cur_chunk_size
                      chunk_start += cur_chunk_size
                      cur_chunk_size = 0
                      budget = chunk_size
  
      with open(out_path, "w", encoding="utf-8", newline="\n") as out_f:
          with multiprocessing.Pool(num_process) as pool:
              for i, (
                  processed_lines,
                  unfiltered_line_num,
              ) in enumerate(
                  pool.imap(self.process_chunk, chunk_iterator())
              ):
                  out_f.write("\n".join(processed_lines) + "\n")
                  filtered_line_count += len(processed_lines)
                  total_line_count += unfiltered_line_num
                  if (i + 1) % 100 == 0:
                      print(
                          "Chunk {}, #Lines Processed: {}, Filtered: {}, Remain: {}".format(
                              i + 1,
                              total_line_count,
                              total_line_count - filtered_line_count,
                              filtered_line_count,
                          )
                      )
      end = time.time()
      print(
          "Done, #Lines {}/{}, Time spent {}".format(
              filtered_line_count, total_line_count, end - start
          )
      )
      return filtered_line_count
  
imports: import argparse
  import os
  import multiprocessing
  import time
  import numpy
  import warnings
  import re
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data.tokenizers import MosesTokenizer, BaseTokenizer, WhitespaceTokenizer, JiebaTokenizer
  from typing import List, Union, Optional
label_desc: Use image node_id 3 for calling the MonoCorpusProcessor obj's underlying member method code with example usage: obj.process_mono_corpus(corpus_paths, out_path, chunk_size, num_process) and returns: filtered_line_count