class_docstr: Adds HuggingFace Datasets https://huggingface.co/datasets/ to DeepPavlov
      
class_docstr_tok: ["Adds","HuggingFace","Datasets","https",":","\/\/huggingface.co\/datasets\/","to","DeepPavlov"]
docstr: Wraps datasets.load_dataset method
  
  Args:
      path: datasets.load_dataset path argument (e.g., `glue`)
      name: datasets.load_dataset name argument (e.g., `mrpc`)
      train: split name to use as training data.
      valid: split name to use as validation data.
      test: split name to use as test data.
  
  Returns:
      Dict[str, List[Dict]]: Dictionary with train, valid, test datasets
docstr_tok: ["Wraps","datasets.load_dataset","method","Args",":","path",":","datasets.load_dataset","path","argument","(","e.g.",",","`","glue","`",")","name",":","datasets.load_dataset","name","argument","(","e.g.",",","`","mrpc","`",")","train",":","split","name","to","use","as","training","data",".","valid",":","split","name","to","use","as","validation","data",".","test",":","split","name","to","use","as","test","data",".","Returns",":","Dict","[","str",",","List","[","Dict","]","]",":","Dictionary","with","train",",","valid",",","test","datasets"]
code: def read(
      self,
      path: str,
      name: Optional[str] = None,
      train: Optional[str] = None,  # for lidirus with no train
      valid: Optional[str] = None,
      test: Optional[str] = None,
      **kwargs,
  ) -> Dict[str, Dataset]:
      """Wraps datasets.load_dataset method
  
      Args:
          path: datasets.load_dataset path argument (e.g., `glue`)
          name: datasets.load_dataset name argument (e.g., `mrpc`)
          train: split name to use as training data.
          valid: split name to use as validation data.
          test: split name to use as test data.
  
      Returns:
          Dict[str, List[Dict]]: Dictionary with train, valid, test datasets
      """
      if "split" in kwargs:
          raise RuntimeError(
              "Split argument was used. Use train, valid, test arguments instead of split."
          )
  
      # pop elements not relevant to BuilderConfig
      downsample_ratio: Union[List[float], float] = kwargs.pop(
          "downsample_ratio", 1.0
      )
      seed = kwargs.pop("seed", 42)
      percentage = kwargs.pop("dev_percentage", 50)
      do_index_correction = kwargs.pop("do_index_correction", True)
  
      split_mapping = {"train": train, "valid": valid, "test": test}
      # filter unused splits
      split_mapping = {
          el: split_mapping[el]
          for el in split_mapping
          if split_mapping[el]
      }
  
      if isinstance(downsample_ratio, float):
          downsample_ratio = [downsample_ratio] * len(split_mapping)
      elif isinstance(downsample_ratio, list) and len(
          downsample_ratio
      ) != len(split_mapping):
          raise ValueError(
              "The number of downsample ratios must be the same as the number of splits"
          )
  
      if path == "russian_super_glue" and "_mixed" in name:
          name = name.replace("_mixed", "")
  
      dataset = load_dataset(
          path=path,
          name=name,
          split=list(split_mapping.values()),
          **kwargs,
      )
  
      if (path == "super_glue" and name == "copa") or (
          path == "russian_super_glue" and name == "parus"
      ):
          lang = "en" if name == "copa" else "ru"
          dataset = [
              dataset_split.map(
                  preprocess_copa,
                  batched=True,
                  fn_kwargs={"lang": lang},
              )
              for dataset_split in dataset
          ]
      elif path == "super_glue" and name == "boolq":
          # danetqa doesn't require the same preprocessing
          dataset = load_dataset(
              path=path,
              name=name,
              split=interleave_splits(
                  splits=list(split_mapping.values()),
                  percentage=percentage,
              ),
              **kwargs,
          )
          dataset = [
              dataset_split.map(preprocess_boolq, batched=True)
              for dataset_split in dataset
          ]
      elif (path == "super_glue" and name == "record") or (
          path == "russian_super_glue" and name == "rucos"
      ):
          label_column = "label"
          dataset = [
              binary_downsample(
                  add_label_names(
                      dataset_split.map(
                          preprocess_record,
                          batched=True,
                          remove_columns=["answers"],
                      ),
                      label_column=label_column,
                      label_names=["False", "True"],
                  ),
                  ratio=ratio,
                  seed=seed,
                  label_column=label_column,
                  do_correction=do_index_correction,
              ).map(add_num_examples, batched=True, batch_size=None)
              for dataset_split, ratio in zip(dataset, downsample_ratio)
          ]
      elif (path == "super_glue" and name == "multirc") or (
          path == "russian_super_glue" and name == "muserc"
      ):
          dataset = [
              dataset_split.map(
                  preprocess_multirc,
                  batched=True,
                  remove_columns=["paragraph", "question"],
              )
              for dataset_split in dataset
          ]
      elif (path == "super_glue" and name == "wsc") or (
          path == "russian_super_glue" and name == "rwsd"
      ):
          dataset = [
              dataset_split.map(
                  preprocess_wsc,
                  batched=True,
                  remove_columns=[
                      "span1_index",
                      "span2_index",
                      "span1_text",
                      "span2_text",
                  ],
              )
              for dataset_split in dataset
          ]
      elif (
          path == "russian_super_glue"
          and name == "terra_mixed"
          and "train" in list(split_mapping.values())
      ):
          tmp_dataset = []
          for d, split in zip(dataset, split_mapping.values()):
              if split == "train":
                  to_mix = load_dataset(
                      "super_glue", "rte", split="train"
                  )
                  combined_train = concatenate_datasets([to_mix, d])
                  tmp_dataset.append(combined_train)
              else:
                  tmp_dataset.append(d)
          dataset = tmp_dataset
  
      elif (
          path == "russian_super_glue"
          and name == "rcb_mixed"
          and "train" in list(split_mapping.values())
      ):
          tmp_dataset = []
          for d, split in zip(dataset, split_mapping.values()):
              if split == "train":
                  to_mix = load_dataset(
                      "super_glue", "cb", split="train"
                  )
                  combined_train = concatenate_datasets(
                      [to_mix, d.remove_columns(["verb", "negation"])]
                  )
                  tmp_dataset.append(combined_train)
              else:
                  tmp_dataset.append(
                      d.remove_columns(["verb", "negation"])
                  )
          dataset = tmp_dataset
      elif (
          path == "russian_super_glue"
          and name == "danetqa_mixed"
          and "train" in list(split_mapping.values())
      ):
          tmp_dataset = []
          for d, split in zip(dataset, split_mapping.values()):
              if split == "train":
                  to_mix = (
                      load_dataset("super_glue", "boolq", split="train")
                      .map(preprocess_boolq, batched=True)
                      .cast(d.features)
                  )
                  combined_train = concatenate_datasets([to_mix, d])
                  tmp_dataset.append(combined_train)
              else:
                  tmp_dataset.append(d)
          dataset = tmp_dataset
      return dict(zip(split_mapping.keys(), dataset))
  
file_path: DeepPavlov/deeppavlov/dataset_readers/huggingface_dataset_reader.py
imports: import re
  from collections import Counter
  from math import floor
  from typing import Dict, Optional, List, Union
  from datasets import load_dataset, Dataset, Features, ClassLabel, concatenate_datasets
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.dataset_reader import DatasetReader
label_desc: Use image node_id 1 for calling the HuggingFaceDatasetReader obj's underlying member method code with example usage: obj.read(path, name, train, valid, test) and returns: dict