


"""simple docstring"""




import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__magic_name__						: Tuple    =				logging.getLogger(__name__)




@dataclass(frozen=lowerCamelCase  )
class 			__snake_case   :
							__a					 =       42
							__a					 =       42
							__a					 =       None
							__a					 =       None
							__a					 =       None




@dataclass(frozen=lowerCamelCase  )
class 			__snake_case   :
							__a					 =       42
							__a					 =       None
							__a					 =       None
							__a					 =       None
							__a					 =       None


if is_torch_available():
					import torch
					from torch.utils.data import Dataset




					class 			__snake_case   (lowerCamelCase  ):
												__a					 =       42




												def __init__(     self:    str     ,							A_:    str     ,							A_:    PreTrainedTokenizer     ,							A_:    str     ,							A_:    Optional[int] = None     ,							A_:    str=False     ,							A_:    bool = False     ,							):
															__lowerCamelCase          =      hans_processors[task]()

															__lowerCamelCase          =      os.path.join(
															    A_     ,							"""cached_{}_{}_{}_{}""".format(
															        """dev""" if evaluate else """train"""     ,							tokenizer.__class__.__name__     ,							str(A_   )     ,							A_     ,							)     ,							)
															__lowerCamelCase          =      processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																		# HACK(label indices are swapped in RoBERTa pretrained model)
																		__lowerCamelCase     ,__lowerCamelCase          =      label_list[2], label_list[1]
															__lowerCamelCase          =      label_list

															# Make sure only the first process in distributed training processes the dataset,
															# and the others will use the cache.
															__lowerCamelCase          =      cached_features_file + """.lock"""
															with FileLock(A_   ):
																		if os.path.exists(A_   ) and not overwrite_cache:
																					logger.info(f'Loading features from cached file {cached_features_file}'   )
																					__lowerCamelCase          =      torch.load(A_   )
																		else:
																					logger.info(f'Creating features from dataset file at {data_dir}'   )

																					__lowerCamelCase          =      (
																					    processor.get_dev_examples(A_   ) if evaluate else processor.get_train_examples(A_   )
																					)

																					logger.info("""Training examples: %s"""     ,							len(A_   )   )
																					__lowerCamelCase          =      hans_convert_examples_to_features(A_     ,							A_     ,							A_     ,							A_   )
																					logger.info("""Saving features into cached file %s"""     ,							A_   )
																					torch.save(self.features     ,							A_   )




												def __len__(     self:    str   ):
															return len(self.features   )




												def __getitem__(     self:    Any     ,							A_:    str   ):
															return self.features[i]




												def   __a   (     self:    Optional[int]   ):
															return self.label_list


if is_tf_available():
					import tensorflow as tf




					class 			__snake_case   :
												__a					 =       42




												def __init__(     self:    Optional[int]     ,							A_:    str     ,							A_:    PreTrainedTokenizer     ,							A_:    str     ,							A_:    Optional[int] = 1_28     ,							A_:    Optional[Any]=False     ,							A_:    bool = False     ,							):
															__lowerCamelCase          =      hans_processors[task]()
															__lowerCamelCase          =      processor.get_labels()
															if tokenizer.__class__ in (
															    RobertaTokenizer,
															    RobertaTokenizerFast,
															    XLMRobertaTokenizer,
															    BartTokenizer,
															    BartTokenizerFast,
															):
																		# HACK(label indices are swapped in RoBERTa pretrained model)
																		__lowerCamelCase     ,__lowerCamelCase          =      label_list[2], label_list[1]
															__lowerCamelCase          =      label_list

															__lowerCamelCase          =      processor.get_dev_examples(A_   ) if evaluate else processor.get_train_examples(A_   )
															__lowerCamelCase          =      hans_convert_examples_to_features(A_     ,							A_     ,							A_     ,							A_   )

															def gen():
																		for ex_index, ex in tqdm.tqdm(enumerate(self.features   )     ,							desc="""convert examples to features"""   ):
																					if ex_index % 1_00_00 == 0:
																								logger.info("""Writing example %d of %d""" % (ex_index, len(A_   ))   )

																					yield (
																					    {
																					        "example_id": 0,
																					        "input_ids": ex.input_ids,
																					        "attention_mask": ex.attention_mask,
																					        "token_type_ids": ex.token_type_ids,
																					    },
																					    ex.label,
																					)

															__lowerCamelCase          =      tf.data.Dataset.from_generator(
															    A_     ,							(
															        {
															            """example_id""": tf.intaa,
															            """input_ids""": tf.intaa,
															            """attention_mask""": tf.intaa,
															            """token_type_ids""": tf.intaa,
															        },
															        tf.intaa,
															    )     ,							(
															        {
															            """example_id""": tf.TensorShape([]   ),
															            """input_ids""": tf.TensorShape([None, None]   ),
															            """attention_mask""": tf.TensorShape([None, None]   ),
															            """token_type_ids""": tf.TensorShape([None, None]   ),
															        },
															        tf.TensorShape([]   ),
															    )     ,							)




												def   __a   (     self:    List[str]   ):
															return self.dataset




												def __len__(     self:    int   ):
															return len(self.features   )




												def __getitem__(     self:    Dict     ,							A_:    int   ):
															return self.features[i]




												def   __a   (     self:    Optional[int]   ):
															return self.label_list




class 			__snake_case   (lowerCamelCase  ):




							def   __a   (     self:    Optional[Any]     ,							A_:    List[str]   ):
										return self._create_examples(self._read_tsv(os.path.join(A_     ,							"""heuristics_train_set.txt"""   )   )     ,							"""train"""   )




							def   __a   (     self:    Dict     ,							A_:    List[str]   ):
										return self._create_examples(self._read_tsv(os.path.join(A_     ,							"""heuristics_evaluation_set.txt"""   )   )     ,							"""dev"""   )




							def   __a   (     self:    Union[str, Any]   ):
										return ["contradiction", "entailment", "neutral"]




							def   __a   (     self:    str     ,							A_:    int     ,							A_:    str   ):
										__lowerCamelCase          =      []
										for i, line in enumerate(A_   ):
													if i == 0:
																continue
													__lowerCamelCase          =      """%s-%s""" % (set_type, line[0])
													__lowerCamelCase          =      line[5]
													__lowerCamelCase          =      line[6]
													__lowerCamelCase          =      line[7][2:] if line[7].startswith("""ex"""   ) else line[7]
													__lowerCamelCase          =      line[0]
													examples.append(InputExample(guid=A_     ,							text_a=A_     ,							text_b=A_     ,							label=A_     ,							pairID=A_   )   )
										return examples





def       a_ (	lowercase__ :List[InputExample],			lowercase__ :List[str],			lowercase__ :int,			lowercase__ :PreTrainedTokenizer,			):
			__lowerCamelCase          =      {label: i for i, label in enumerate(lowercase__     )}

			__lowerCamelCase          =      []
			for ex_index, example in tqdm.tqdm(enumerate(lowercase__     ),			desc="""convert examples to features"""     ):
						if ex_index % 1_0000 == 0:
									logger.info("""Writing example %d""" % (ex_index)     )

						__lowerCamelCase          =      tokenizer(
						    example.text_a,			example.text_b,			add_special_tokens=lowercase__,			max_length=lowercase__,			padding="""max_length""",			truncation=lowercase__,			return_overflowing_tokens=lowercase__,			)

						__lowerCamelCase          =      label_map[example.label] if example.label in label_map else 0

						__lowerCamelCase          =      int(example.pairID     )

						features.append(InputFeatures(**lowercase__,			label=lowercase__,			pairID=lowercase__     )     )

			for i, example in enumerate(examples[:5]     ):
						logger.info("""*** Example ***"""     )
						logger.info(f'guid: {example}'     )
						logger.info(f'features: {features[i]}'     )

			return features


__magic_name__						: List[str]    =				{
    'hans': 3,
}


__magic_name__						: Dict    =				{
    'hans': HansProcessor,
}








"""simple docstring"""




import json
import os
import tempfile

from transformers.testing_utils import check_json_file_has_correct_format




class 			__snake_case   :
							__a					 =       None




							def   __a   (     self:    int   ):
										__lowerCamelCase          =      self.feature_extraction_class(**self.feat_extract_dict   )
										__lowerCamelCase          =      json.loads(feat_extract.to_json_string()   )
										for key, value in self.feat_extract_dict.items():
													self.assertEqual(obj[key]     ,							A_   )




							def   __a   (     self:    List[str]   ):
										__lowerCamelCase          =      self.feature_extraction_class(**self.feat_extract_dict   )

										with tempfile.TemporaryDirectory() as tmpdirname:
													__lowerCamelCase          =      os.path.join(A_     ,							"""feat_extract.json"""   )
													feat_extract_first.to_json_file(A_   )
													__lowerCamelCase          =      self.feature_extraction_class.from_json_file(A_   )

										self.assertEqual(feat_extract_second.to_dict()     ,							feat_extract_first.to_dict()   )




							def   __a   (     self:    Optional[Any]   ):
										__lowerCamelCase          =      self.feature_extraction_class(**self.feat_extract_dict   )

										with tempfile.TemporaryDirectory() as tmpdirname:
													__lowerCamelCase          =      feat_extract_first.save_pretrained(A_   )[0]
													check_json_file_has_correct_format(A_   )
													__lowerCamelCase          =      self.feature_extraction_class.from_pretrained(A_   )

										self.assertEqual(feat_extract_second.to_dict()     ,							feat_extract_first.to_dict()   )




							def   __a   (     self:    List[Any]   ):
										__lowerCamelCase          =      self.feature_extraction_class()
										self.assertIsNotNone(A_   )




