class_docstr: Process a pair of corpus.
  
  This largely recovers the functionality of 'clean-corpus-n.perl' in mosesdecoder.
  The difference is that it is customizable with pure python.
  
  By default, we will perform the following pre-processing pipeline.
  Each stage could be turned on/off and specialized based on the input arguments.
  Also, you may directly revise the code and write your own processing script.
  
  1. Normalize sentence
  2. Pre-filter
  3. Tokenize the sentence
  4. Filter the sentence based on different rules
      3.1 Remove pairs where `max(len(lhs) / len(rhs), len(rhs) / len(lhs) > max_ratio`
      3.2 Remove pairs where not `min_max_words <= len(lhs) <= max_num_words` and
                                 `min_max_words <= len(rhs) <= max_num_words`
class_docstr_tok: ["Process","a","pair","of","corpus",".","This","largely","recovers","the","functionality","of","'clean-corpus-n.perl","'","in","mosesdecoder",".","The","difference","is","that","it","is","customizable","with","pure","python",".","By","default",",","we","will","perform","the","following","pre-processing","pipeline",".","Each","stage","could","be","turned","on\/off","and","specialized","based","on","the","input","arguments",".","Also",",","you","may","directly","revise","the","code","and","write","your","own","processing","script",".","1",".","Normalize","sentence","2",".","Pre-filter","3",".","Tokenize","the","sentence","4",".","Filter","the","sentence","based","on","different","rules","3.1","Remove","pairs","where","`","max","(","len","(","lhs",")","\/","len","(","rhs",")",",","len","(","rhs",")","\/","len","(","lhs",")",">","max_ratio","`","3.2","Remove","pairs","where","not","`","min_max_words","<","=","len","(","lhs",")","<","=","max_num_words","`","and","`","min_max_words","<","=","len","(","rhs",")","<","=","max_num_words","`"]
docstr: Preprocess the parallel corpus
  
  Parameters
  ----------
  src_corpus_paths
      Source corpus paths
  tgt_corpus_paths
      Target corpus paths
  src_out_path
      Write the results to the source output path
  tgt_out_path
      Write the results to the target output path
  chunk_size
      Approximately split the corpus files into multiple chunks
  num_process
      The number of process
  
  Returns
  -------
  line_count
      The number of lines in the final filtered file
docstr_tok: ["Preprocess","the","parallel","corpus","Parameters","--","--","--","--","--","src_corpus_paths","Source","corpus","paths","tgt_corpus_paths","Target","corpus","paths","src_out_path","Write","the","results","to","the","source","output","path","tgt_out_path","Write","the","results","to","the","target","output","path","chunk_size","Approximately","split","the","corpus","files","into","multiple","chunks","num_process","The","number","of","process","Returns","--","--","--","-","line_count","The","number","of","lines","in","the","final","filtered","file"]
code: def process_parallel_corpus(
      self,
      src_corpus_paths: List[str],
      tgt_corpus_paths: List[str],
      src_out_path: str,
      tgt_out_path: str,
      chunk_size: int = 1024 * 1024,
      num_process: int = 8,
  ) -> int:
      """Preprocess the parallel corpus
  
      Parameters
      ----------
      src_corpus_paths
          Source corpus paths
      tgt_corpus_paths
          Target corpus paths
      src_out_path
          Write the results to the source output path
      tgt_out_path
          Write the results to the target output path
      chunk_size
          Approximately split the corpus files into multiple chunks
      num_process
          The number of process
  
      Returns
      -------
      line_count
          The number of lines in the final filtered file
      """
      start = time.time()
      total_line_count = 0
      filtered_line_count = 0
  
      def chunk_iterator(step=10):
          for src_path, tgt_path in zip(
              src_corpus_paths, tgt_corpus_paths
          ):
              src_line_pos = get_line_byte_start(src_path)
              tgt_line_pos = get_line_byte_start(tgt_path)
              src_line_size = src_line_pos[1:] - src_line_pos[:-1]
              tgt_line_size = tgt_line_pos[1:] - tgt_line_pos[:-1]
              num_src_lines = src_line_pos.shape[0] - 1
              num_tgt_lines = tgt_line_pos.shape[0] - 1
              assert num_src_lines == num_tgt_lines
              src_budget = chunk_size
              tgt_budget = chunk_size
              src_chunk_start = 0
              tgt_chunk_start = 0
              src_chunk_size = 0
              tgt_chunk_size = 0
              for i in range(0, num_src_lines, step):
                  line_batch_num = min(num_src_lines - i, step)
                  src_batch_line_size = src_line_size[
                      i : (i + line_batch_num)
                  ].sum()
                  tgt_batch_line_size = tgt_line_size[
                      i : (i + line_batch_num)
                  ].sum()
                  src_budget -= src_batch_line_size
                  tgt_budget -= tgt_batch_line_size
                  src_chunk_size += src_batch_line_size
                  tgt_chunk_size += tgt_batch_line_size
                  if (
                      src_budget <= 0
                      or tgt_budget <= 0
                      or i + step >= num_src_lines
                  ):
                      yield src_path, src_chunk_start, src_chunk_size, tgt_path, tgt_chunk_start, tgt_chunk_size
                      src_chunk_start += src_chunk_size
                      tgt_chunk_start += tgt_chunk_size
                      src_chunk_size = 0
                      tgt_chunk_size = 0
                      src_budget = chunk_size
                      tgt_budget = chunk_size
  
      with open(
          src_out_path, "w", encoding="utf-8", newline="\n"
      ) as src_out_f:
          with open(
              tgt_out_path, "w", encoding="utf-8", newline="\n"
          ) as tgt_out_f:
              with multiprocessing.Pool(num_process) as pool:
                  for i, (
                      processed_src_lines,
                      processed_tgt_lines,
                      unfiltered_line_num,
                  ) in enumerate(
                      pool.imap(self.process_chunk, chunk_iterator())
                  ):
                      src_out_f.write(
                          "\n".join(processed_src_lines) + "\n"
                      )
                      tgt_out_f.write(
                          "\n".join(processed_tgt_lines) + "\n"
                      )
                      filtered_line_count += len(processed_src_lines)
                      total_line_count += unfiltered_line_num
                      if (i + 1) % 100 == 0:
                          print(
                              "Chunk {}, #Lines Processed: {}, Filtered: {}, Remain: {}".format(
                                  i + 1,
                                  total_line_count,
                                  total_line_count
                                  - filtered_line_count,
                                  filtered_line_count,
                              )
                          )
      end = time.time()
      print(
          "Done, #Lines {}/{}, Time spent {}".format(
              filtered_line_count, total_line_count, end - start
          )
      )
      return filtered_line_count
  
imports: import argparse
  import os
  import multiprocessing
  import time
  import numpy
  import warnings
  import re
  from gluonnlp.data.filtering import MosesNormalizer
  from gluonnlp.data.tokenizers import MosesTokenizer, BaseTokenizer, WhitespaceTokenizer, JiebaTokenizer
  from typing import List, Union, Optional
label_desc: Use image node_id 3 for calling the ParallelCorpusProcessor obj's underlying member method code with example usage: obj.process_parallel_corpus(src_corpus_paths, tgt_corpus_paths, src_out_path, tgt_out_path, chunk_size, num_process) and returns: filtered_line_count