class_docstr: Clean examples according to rules
docstr: Convert source and target examples to doc level segments.
docstr_tok: ["Convert","source","and","target","examples","to","doc","level","segments","."]
code: def batch_apply(self, batch, is_train=False, stats=None, **kwargs):
      """Convert source and target examples to doc level segments."""
  
      def _id(string):
          res = self.id_func.predict(string, k=1)
          res = res[0][0].replace("__label__", "")
          return res
  
      trf_batch = []
  
      for ex, _, cid in batch:
          if self.scripts_ok_dict[cid]:
              ok_regex = (
                  "[^"
                  + "".join(
                      r"\p{%s}" % sc for sc in self.scripts_ok_dict[cid]
                  )
                  + "]"
              )
  
          if self.scripts_nok_dict[cid]:
              nok_regex = (
                  "["
                  + "".join(
                      r"\p{%s}" % sc
                      for sc in self.scripts_nok_dict[cid]
                  )
                  + "]"
              )
  
          src_str = " ".join(ex["src"])
          if len(src_str) == 0:
              # print("src empty")
              continue
          if self.same_char_dict[cid] and re.search(
              r"([^0-9])\1{3}", src_str
          ):
              # print("too many same char in src")
              continue
          if self.same_word_dict[cid] and re.search(
              r"(\ .*|.*\ )\1{2}", src_str
          ):
              # print("too many same word in src")
              continue
          if len(src_str) / len(ex["src"]) < self.avg_tok_min_dict[cid]:
              # print("avg token min", len(src_str) / len(ex['src']))
              continue
          if len(src_str) / len(ex["src"]) > self.avg_tok_max_dict[cid]:
              # print("avg token max", len(src_str) / len(ex['src']))
              continue
  
          if self.scripts_ok_dict[cid] and re.search(ok_regex, src_str):
              # print("text does not fully belong to wanted script")
              continue
          if self.scripts_nok_dict[cid] and re.search(
              nok_regex, src_str
          ):
              # print("Some text belong to unwanted scripts")
              continue
  
          if (
              self.langid_dict[cid] != []
              and _id(src_str) not in self.langid_dict[cid]
          ):
              # print("langid does not match", _id(src_str))
              continue
  
          if ex["tgt"] is not None:
              tgt_str = " ".join(ex["tgt"])
              if self.src_eq_tgt_dict[cid] and src_str == tgt_str:
                  # print("src = tgt")
                  continue
              if len(tgt_str) == 0:
                  # print("tgt empty")
                  continue
              if (len(ex["src"]) + 1) / (
                  len(ex["tgt"]) + 1
              ) > self.src_tgt_ratio_dict[cid] or (
                  len(ex["src"]) + 1
              ) / (
                  len(ex["tgt"]) + 1
              ) < (
                  1 / self.src_tgt_ratio_dict[cid]
              ):
                  # print("src / tgt ratio ", len(src_str) / len(tgt_str))
                  continue
              if self.same_char_dict[cid] and re.search(
                  r"([^0-9])\1{3}", tgt_str
              ):
                  # print("too many same char in tgt")
                  continue
              if self.same_word_dict[cid] and re.search(
                  r"(\ .*|.*\ )\1{2}", tgt_str
              ):
                  # print("too many same word in tgt")
                  continue
              if (
                  len(tgt_str) / len(ex["tgt"])
                  < self.avg_tok_min_dict[cid]
              ):
                  # print("avg token min", len(tgt_str) / len(ex['tgt']))
                  continue
              if (
                  len(tgt_str) / len(ex["tgt"])
                  > self.avg_tok_max_dict[cid]
              ):
                  # print("avg token max", len(tgt_str) / len(ex['tgt']))
                  continue
  
              if self.scripts_ok_dict[cid] and re.search(
                  ok_regex, tgt_str
              ):
                  # print("text does not fully belong to wanted script")
                  continue
              if self.scripts_nok_dict[cid] and re.search(
                  nok_regex, tgt_str
              ):
                  # print("Some text belong to unwanted scripts")
                  continue
  
              if (
                  self.langid_dict[cid] != []
                  and _id(tgt_str) not in self.langid_dict[cid]
              ):
                  # print("langid does not match", _id(tgt_str))
                  continue
  
          trf_batch.append((ex, self, cid))
  
      return trf_batch
  
file_path: OpenNMT-py/onmt/transforms/clean.py
imports: from onmt.transforms import register_transform
  from .transform import Transform
  from onmt.utils.logging import logger
  import fasttext
  import os
  import urllib.request
  import regex
label_desc: Use image node_id 7 for calling the CleanTransform obj's underlying member method code with example usage: obj.batch_apply(batch, is_train, stats) and returns: trf_batch, res