docstr: Cleans source and target sentences to get rid of noisy data.
  Specifically, a pair of sentences is removed if
    -- either source or target is longer than *max_tokens*
    -- either source or target is shorter than *min_tokens*
    -- absolute difference between source and target is larger than
       *max_tokens_diff*
    -- one sentence is *max_tokens_ratio* times longer than the other
docstr_tok: ["Cleans","source","and","target","sentences","to","get","rid","of","noisy","data",".","Specifically",",","a","pair","of","sentences","is","removed","if","--","either","source","or","target","is","longer","than","*","max_tokens","*","--","either","source","or","target","is","shorter","than","*","min_tokens","*","--","absolute","difference","between","source","and","target","is","larger","than","*","max_tokens_diff","*","--","one","sentence","is","*","max_tokens_ratio","*","times","longer","than","the","other"]
code: def clean_src_and_target(
      self,
      src_ids,
      tgt_ids,
      max_tokens=None,
      min_tokens=None,
      max_tokens_diff=None,
      max_tokens_ratio=None,
      filter_equal_src_and_dest=False,
  ):
      """
      Cleans source and target sentences to get rid of noisy data.
      Specifically, a pair of sentences is removed if
        -- either source or target is longer than *max_tokens*
        -- either source or target is shorter than *min_tokens*
        -- absolute difference between source and target is larger than
           *max_tokens_diff*
        -- one sentence is *max_tokens_ratio* times longer than the other
      """
  
      if len(src_ids) != len(tgt_ids):
          raise ValueError(
              "Source and target corpora have different lengths!"
          )
      src_ids_, tgt_ids_ = [], []
      for i in range(len(src_ids)):
          src_len, tgt_len = len(src_ids[i]), len(tgt_ids[i])
          if (
              (
                  max_tokens is not None
                  and (src_len > max_tokens or tgt_len > max_tokens)
              )
              or (
                  min_tokens is not None
                  and (src_len < min_tokens or tgt_len < min_tokens)
              )
              or (
                  filter_equal_src_and_dest and src_ids[i] == tgt_ids[i]
              )
              or (
                  max_tokens_diff is not None
                  and np.abs(src_len - tgt_len) > max_tokens_diff
              )
          ):
              continue
          if max_tokens_ratio is not None:
              ratio = max(src_len - 2, 1) / max(tgt_len - 2, 1)
              if ratio > max_tokens_ratio or ratio < (
                  1 / max_tokens_ratio
              ):
                  continue
          src_ids_.append(src_ids[i])
          tgt_ids_.append(tgt_ids[i])
      return src_ids_, tgt_ids_
  
imports: import io
  import json
  import pickle
  from collections import OrderedDict
  from dataclasses import dataclass
  from typing import Any, List, Optional
  import braceexpand
  import numpy
  import webdataset
  from torch.utils.data import IterableDataset
  from nemo.collections.nlp.data.data_utils.data_preprocessing import dataset_to_ids
  from nemo.core import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 7 for calling the TranslationDataset obj's underlying member method code with example usage: obj.clean_src_and_target(src_ids, tgt_ids, max_tokens, min_tokens, max_tokens_diff, max_tokens_ratio, filter_equal_src_and_dest) and returns: src_ids_,  tgt_ids_