



'''simple docstring'''





import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__a  			=						logging.getLogger(__name__)







@dataclass(frozen=_a      )
class        UpperCAmelCase_			:
    """simple docstring"""







    lowercase   =						42
    lowercase   =						42
    lowercase   =						None
    lowercase   =						None
    lowercase   =						None







@dataclass(frozen=_a      )
class        UpperCAmelCase_			:
    """simple docstring"""







    lowercase   =						42
    lowercase   =						None
    lowercase   =						None
    lowercase   =						None
    lowercase   =						None


if is_torch_available():
       import torch
       from torch.utils.data import Dataset







       class        UpperCAmelCase_			(			_a      ):
           """simple docstring"""







           lowercase   =						42




           def __init__(       self   :	Optional[Any]					,	snake_case_   :	str					,	snake_case_   :	PreTrainedTokenizer					,	snake_case_   :	str					,	snake_case_   :	Optional[int] = None					,	snake_case_   :	Optional[Any]=False					,	snake_case_   :	bool = False					,	):
                 snake_case__    :       Any       		= hans_processors[task]()

                 snake_case__    :       Optional[int]       		= os.path.join(
                     snake_case_					,	"""cached_{}_{}_{}_{}""".format(
                         """dev""" if evaluate else """train"""					,	tokenizer.__class__.__name__					,	str(snake_case_		)					,	snake_case_					,	)					,	)
                 snake_case__    :       Union[str, Any]       		= processor.get_labels()
                 if tokenizer.__class__ in (
                     RobertaTokenizer,
                     RobertaTokenizerFast,
                     XLMRobertaTokenizer,
                     BartTokenizer,
                     BartTokenizerFast,
                 ):
                       # HACK(label indices are swapped in RoBERTa pretrained model)
                       snake_case__		,							snake_case__    :       List[str]       		= label_list[2], label_list[1]
                 snake_case__    :       Optional[Any]       		= label_list

                 # Make sure only the first process in distributed training processes the dataset,
                 # and the others will use the cache.
                 snake_case__    :       Tuple       		= cached_features_file + """.lock"""
                 with FileLock(snake_case_		):
                       if os.path.exists(snake_case_		) and not overwrite_cache:
                             logger.info(f"Loading features from cached file {cached_features_file}"		)
                             snake_case__    :       Union[str, Any]       		= torch.load(snake_case_		)
                       else:
                             logger.info(f"Creating features from dataset file at {data_dir}"		)

                             snake_case__    :       List[Any]       		= (
                                 processor.get_dev_examples(snake_case_		) if evaluate else processor.get_train_examples(snake_case_		)
                             )

                             logger.info("""Training examples: %s"""					,	len(snake_case_		)		)
                             snake_case__    :       Dict       		= hans_convert_examples_to_features(snake_case_					,	snake_case_					,	snake_case_					,	snake_case_		)
                             logger.info("""Saving features into cached file %s"""					,	snake_case_		)
                             torch.save(self.features					,	snake_case_		)




           def __len__(       self   :	Tuple		):
                 return len(self.features		)




           def __getitem__(       self   :	Optional[int]					,	snake_case_   :	str		):
                 return self.features[i]




           def lowerCamelCase			(       self   :	Dict		):
                 return self.label_list


if is_tf_available():
       import tensorflow as tf







       class        UpperCAmelCase_			:
           """simple docstring"""







           lowercase   =						42




           def __init__(       self   :	Any					,	snake_case_   :	str					,	snake_case_   :	PreTrainedTokenizer					,	snake_case_   :	str					,	snake_case_   :	Optional[int] = 128					,	snake_case_   :	int=False					,	snake_case_   :	bool = False					,	):
                 snake_case__    :       str       		= hans_processors[task]()
                 snake_case__    :       Any       		= processor.get_labels()
                 if tokenizer.__class__ in (
                     RobertaTokenizer,
                     RobertaTokenizerFast,
                     XLMRobertaTokenizer,
                     BartTokenizer,
                     BartTokenizerFast,
                 ):
                       # HACK(label indices are swapped in RoBERTa pretrained model)
                       snake_case__		,							snake_case__    :       Tuple       		= label_list[2], label_list[1]
                 snake_case__    :       Optional[Any]       		= label_list

                 snake_case__    :       str       		= processor.get_dev_examples(snake_case_		) if evaluate else processor.get_train_examples(snake_case_		)
                 snake_case__    :       List[Any]       		= hans_convert_examples_to_features(snake_case_					,	snake_case_					,	snake_case_					,	snake_case_		)

                 def gen():
                       for ex_index, ex in tqdm.tqdm(enumerate(self.features		)					,	desc="""convert examples to features"""		):
                             if ex_index % 10_000 == 0:
                                   logger.info("""Writing example %d of %d""" % (ex_index, len(snake_case_		))		)

                             yield (
                                 {
                                     "example_id": 0,
                                     "input_ids": ex.input_ids,
                                     "attention_mask": ex.attention_mask,
                                     "token_type_ids": ex.token_type_ids,
                                 },
                                 ex.label,
                             )

                 snake_case__    :       Dict       		= tf.data.Dataset.from_generator(
                     snake_case_					,	(
                         {
                             """example_id""": tf.intaa,
                             """input_ids""": tf.intaa,
                             """attention_mask""": tf.intaa,
                             """token_type_ids""": tf.intaa,
                         },
                         tf.intaa,
                     )					,	(
                         {
                             """example_id""": tf.TensorShape([]		),
                             """input_ids""": tf.TensorShape([None, None]		),
                             """attention_mask""": tf.TensorShape([None, None]		),
                             """token_type_ids""": tf.TensorShape([None, None]		),
                         },
                         tf.TensorShape([]		),
                     )					,	)




           def lowerCamelCase			(       self   :	Optional[Any]		):
                 return self.dataset




           def __len__(       self   :	Dict		):
                 return len(self.features		)




           def __getitem__(       self   :	Union[str, Any]					,	snake_case_   :	List[Any]		):
                 return self.features[i]




           def lowerCamelCase			(       self   :	List[str]		):
                 return self.label_list







class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    def lowerCamelCase			(       self   :	int					,	snake_case_   :	List[str]		):
          return self._create_examples(self._read_tsv(os.path.join(snake_case_					,	"""heuristics_train_set.txt"""		)		)					,	"""train"""		)




    def lowerCamelCase			(       self   :	Union[str, Any]					,	snake_case_   :	Optional[Any]		):
          return self._create_examples(self._read_tsv(os.path.join(snake_case_					,	"""heuristics_evaluation_set.txt"""		)		)					,	"""dev"""		)




    def lowerCamelCase			(       self   :	Tuple		):
          return ["contradiction", "entailment", "neutral"]




    def lowerCamelCase			(       self   :	Union[str, Any]					,	snake_case_   :	int					,	snake_case_   :	Any		):
          snake_case__    :       Dict       		= []
          for i, line in enumerate(snake_case_		):
                if i == 0:
                      continue
                snake_case__    :       int       		= """%s-%s""" % (set_type, line[0])
                snake_case__    :       Tuple       		= line[5]
                snake_case__    :       Tuple       		= line[6]
                snake_case__    :       Any       		= line[7][2:] if line[7].startswith("""ex"""		) else line[7]
                snake_case__    :       List[Any]       		= line[0]
                examples.append(InputExample(guid=snake_case_					,	text_a=snake_case_					,	text_b=snake_case_					,	label=snake_case_					,	pairID=snake_case_		)		)
          return examples





def 		__snake_case(							_lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase	,  )						->					Tuple:
      snake_case__    :       Optional[Any]       		= {label: i for i, label in enumerate(_lowerCAmelCase						)}

      snake_case__    :       Any       		= []
      for ex_index, example in tqdm.tqdm(enumerate(_lowerCAmelCase						)	,  desc="""convert examples to features"""						):
            if ex_index % 10_000 == 0:
                  logger.info("""Writing example %d""" % (ex_index)						)

            snake_case__    :       Optional[Any]       		= tokenizer(
                example.text_a	,  example.text_b	,  add_special_tokens=_lowerCAmelCase	,  max_length=_lowerCAmelCase	,  padding="""max_length"""	,  truncation=_lowerCAmelCase	,  return_overflowing_tokens=_lowerCAmelCase	,  )

            snake_case__    :       List[Any]       		= label_map[example.label] if example.label in label_map else 0

            snake_case__    :       Tuple       		= int(example.pairID						)

            features.append(InputFeatures(**_lowerCAmelCase	,  label=_lowerCAmelCase	,  pairID=_lowerCAmelCase						)						)

      for i, example in enumerate(examples[:5]						):
            logger.info("""*** Example ***"""						)
            logger.info(f"guid: {example}"						)
            logger.info(f"features: {features[i]}"						)

      return features


__a  			=						{
    "hans": 3,
}



__a  			=						{
    "hans": HansProcessor,
}





'''simple docstring'''





import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


__a  			=						[
    {"dataset": "wikipedia", "config_name": "20220301.de"},
    {"dataset": "wikipedia", "config_name": "20220301.en"},
    {"dataset": "wikipedia", "config_name": "20220301.fr"},
    {"dataset": "wikipedia", "config_name": "20220301.frr"},
    {"dataset": "wikipedia", "config_name": "20220301.it"},
    {"dataset": "wikipedia", "config_name": "20220301.simple"},
    {"dataset": "snli", "config_name": "plain_text"},
    {"dataset": "eli5", "config_name": "LFQA_reddit"},
    {"dataset": "wiki40b", "config_name": "en"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.compressed"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.nq.no_index"},
    {"dataset": "wiki_dpr", "config_name": "psgs_w100.multiset.no_index"},
    {"dataset": "natural_questions", "config_name": "default"},
]





def 		__snake_case(							_lowerCAmelCase=True						)						->					Dict:
      if with_config:
            return [
                {
                    "testcase_name": d["dataset"] + "/" + d["config_name"],
                    "dataset": d["dataset"],
                    "config_name": d["config_name"],
                }
                for d in DATASETS_ON_HF_GCP
            ]
      else:
            return [
                {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
            ]







@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=_a      )      )
class        UpperCAmelCase_			(			_a      ):
    """simple docstring"""







    lowercase   =						None
    lowercase   =						None




    def lowerCamelCase			(       self   :	List[Any]					,	snake_case_   :	Union[str, Any]					,	snake_case_   :	Optional[int]		):
          with TemporaryDirectory() as tmp_dir:
                snake_case__    :       Dict       		= dataset_module_factory(snake_case_					,	cache_dir=snake_case_		)

                snake_case__    :       Optional[int]       		= import_main_class(dataset_module.module_path					,	dataset=snake_case_		)

                snake_case__    :       DatasetBuilder       		= builder_cls(
                    cache_dir=snake_case_					,	config_name=snake_case_					,	hash=dataset_module.hash					,	)

                snake_case__    :       Dict       		= """/""".join(
                    [
                        HF_GCP_BASE_URL,
                        builder_instance._relative_data_dir(with_hash=snake_case_		).replace(os.sep					,	"""/"""		),
                        config.DATASET_INFO_FILENAME,
                    ]		)
                snake_case__    :       List[str]       		= cached_path(snake_case_					,	cache_dir=snake_case_		)
                self.assertTrue(os.path.exists(snake_case_		)		)





@pytest.mark.integration
def 		__snake_case(							_lowerCAmelCase						)						->					Tuple:
      snake_case__    :       Optional[int]       		= tmp_path_factory.mktemp("""test_hf_gcp"""						) / """test_wikipedia_simple"""
      snake_case__    :       Dict       		= dataset_module_factory("""wikipedia"""	,  cache_dir=_lowerCAmelCase						)
      snake_case__    :       Dict       		= import_main_class(dataset_module.module_path						)
      snake_case__    :       DatasetBuilder       		= builder_cls(
          cache_dir=_lowerCAmelCase	,  config_name="""20220301.frr"""	,  hash=dataset_module.hash	,  )
      # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
      snake_case__    :       Any       		= None
      builder_instance.download_and_prepare()
      snake_case__    :       Union[str, Any]       		= builder_instance.as_dataset()
      assert ds





@pytest.mark.integration
def 		__snake_case(							_lowerCAmelCase						)						->					List[str]:
      snake_case__    :       Optional[int]       		= dataset_module_factory("""wikipedia"""	,  cache_dir=_lowerCAmelCase						)
      snake_case__    :       List[str]       		= import_main_class(dataset_module.module_path	,  dataset=_lowerCAmelCase						)
      snake_case__    :       DatasetBuilder       		= builder_cls(
          cache_dir=_lowerCAmelCase	,  config_name="""20220301.frr"""	,  hash=dataset_module.hash	,  )
      snake_case__    :       Any       		= builder_instance.as_streaming_dataset()
      assert ds
      assert isinstance(_lowerCAmelCase	,  _lowerCAmelCase						)
      assert "train" in ds
      assert isinstance(ds["""train"""]	,  _lowerCAmelCase						)
      assert next(iter(ds["""train"""]						)						)
