docstr: Cleans source and target sentences to get rid of noisy data.
  Specifically, a pair of sentences is removed if
    -- either source or target is longer than *max_tokens*
    -- either source or target is shorter than *min_tokens*
    -- absolute difference between source and target is larger than
       *max_tokens_diff*
    -- one sentence is *max_tokens_ratio* times longer than the other
docstr_tok: ["Cleans","source","and","target","sentences","to","get","rid","of","noisy","data",".","Specifically",",","a","pair","of","sentences","is","removed","if","--","either","source","or","target","is","longer","than","*","max_tokens","*","--","either","source","or","target","is","shorter","than","*","min_tokens","*","--","absolute","difference","between","source","and","target","is","larger","than","*","max_tokens_diff","*","--","one","sentence","is","*","max_tokens_ratio","*","times","longer","than","the","other"]
code: def clean(self, ids, max_tokens=None, min_tokens=None):
      """
      Cleans source and target sentences to get rid of noisy data.
      Specifically, a pair of sentences is removed if
        -- either source or target is longer than *max_tokens*
        -- either source or target is shorter than *min_tokens*
        -- absolute difference between source and target is larger than
           *max_tokens_diff*
        -- one sentence is *max_tokens_ratio* times longer than the other
      """
  
      ids_ = []
      for i in range(len(ids)):
          len_ = len(ids[i])
          if (max_tokens is not None and len_ > max_tokens) or (
              min_tokens is not None and len_ < min_tokens
          ):
              continue
          ids_.append(ids[i])
      return ids_
  
imports: import io
  import json
  import logging
  import pickle
  from collections import OrderedDict
  from typing import Any
  import braceexpand
  import numpy
  import webdataset
  from torch.utils.data import IterableDataset
  from nemo.collections.nlp.data.data_utils.data_preprocessing import dataset_to_ids
  from nemo.core import Dataset
label_desc: Use image node_id 6 for calling the SentenceDataset obj's underlying member method code with example usage: obj.clean(ids, max_tokens, min_tokens) and returns: ids_