code: def filter(example, uniques, args):
      """Filter dataset with heuristics. Config, test and has_no_keywords files are removed with a given probability."""
      if not check_uniques(example, uniques):
          return False
      elif example["autogenerated"]:
          return False
      elif example["line_max"] > args.line_max:
          return False
      elif example["line_mean"] > args.line_mean:
          return False
      elif example["alpha_frac"] < args.alpha_frac:
          return False
      elif example["ratio"] < args.min_token_ratio:
          return False
      elif (
          example["config_or_test"]
          and np.random.rand() <= args.filter_proba
      ):
          return False
      elif (
          example["has_no_keywords"]
          and np.random.rand() <= args.filter_proba
      ):
          return False
      elif example["has_few_assignments"]:
          return False
      else:
          return True
  
imports: import gzip
  import hashlib
  import json
  import multiprocessing
  import os
  import re
  import shutil
  import time
  from pathlib import Path
  import numpy
  from datasets import load_dataset
  from arguments import PreprocessingArguments
  from minhash_deduplication import deduplicate_dataset
  from transformers import AutoTokenizer, HfArgumentParser
label_desc: Use image node_id 11 for calling a global function with example usage: filter(example, uniques, args) and returns: False, False, False, False, False, False, False, False, False, True