





'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowerCAmelCase									=   logging.getLogger(__name__)
@dataclass(frozen=_A							)
class 		lowerCamelCase	:
	snake_case_   = 42
	snake_case_   = 42
	snake_case_   = None
	snake_case_   = None
	snake_case_   = None






@dataclass(frozen=_A							)
class 		lowerCamelCase	:
	snake_case_   = 42
	snake_case_   = None
	snake_case_   = None
	snake_case_   = None
	snake_case_   = None


if is_torch_available():
					import torch
					from torch.utils.data import Dataset
					class 		lowerCamelCase	(   _A							):
						snake_case_   = 42



						def __init__( self		,   a_		,   a_		,   a_		,   a_ = None		,   a_=False		,   a_ = False		,   ):
								lowerCAmelCase  :     Union[str, Any]       						= hans_processors[task]()

								lowerCAmelCase  :     str       						= os.path.join(
								    a_		,   "cached_{}_{}_{}_{}".format(
								        "dev" if evaluate else "train"		,   tokenizer.__class__.__name__		,   str(a_						)		,   a_		,   )		,   )
								lowerCAmelCase  :     Any       						= processor.get_labels()
								if tokenizer.__class__ in (
								    RobertaTokenizer,
								    RobertaTokenizerFast,
								    XLMRobertaTokenizer,
								    BartTokenizer,
								    BartTokenizerFast,
								):
										# HACK(label indices are swapped in RoBERTa pretrained model)
										lowerCAmelCase						,		lowerCAmelCase  :     Tuple       						= label_list[2], label_list[1]
								lowerCAmelCase  :     List[Any]       						= label_list

								# Make sure only the first process in distributed training processes the dataset,
								# and the others will use the cache.
								lowerCAmelCase  :     Tuple       						= cached_features_file + ".lock"
								with FileLock(a_						):
										if os.path.exists(a_						) and not overwrite_cache:
												logger.info(F'''Loading features from cached file {cached_features_file}'''						)
												lowerCAmelCase  :     Tuple       						= torch.load(a_						)
										else:
												logger.info(F'''Creating features from dataset file at {data_dir}'''						)

												lowerCAmelCase  :     Tuple       						= (
												    processor.get_dev_examples(a_						) if evaluate else processor.get_train_examples(a_						)
												)

												logger.info("Training examples: %s"		,   len(a_						)						)
												lowerCAmelCase  :     int       						= hans_convert_examples_to_features(a_		,   a_		,   a_		,   a_						)
												logger.info("Saving features into cached file %s"		,   a_						)
												torch.save(self.features		,   a_						)



						def __len__( self						):
								return len(self.features						)



						def __getitem__( self		,   a_						):
								return self.features[i]





						def 	_lowerCamelCase     ( self						):
								return self.label_list


if is_tf_available():
					import tensorflow as tf
					class 		lowerCamelCase	:
						snake_case_   = 42



						def __init__( self		,   a_		,   a_		,   a_		,   a_ = 128		,   a_=False		,   a_ = False		,   ):
								lowerCAmelCase  :     Dict       						= hans_processors[task]()
								lowerCAmelCase  :     Union[str, Any]       						= processor.get_labels()
								if tokenizer.__class__ in (
								    RobertaTokenizer,
								    RobertaTokenizerFast,
								    XLMRobertaTokenizer,
								    BartTokenizer,
								    BartTokenizerFast,
								):
										# HACK(label indices are swapped in RoBERTa pretrained model)
										lowerCAmelCase						,		lowerCAmelCase  :     Optional[int]       						= label_list[2], label_list[1]
								lowerCAmelCase  :     Any       						= label_list

								lowerCAmelCase  :     Any       						= processor.get_dev_examples(a_						) if evaluate else processor.get_train_examples(a_						)
								lowerCAmelCase  :     Optional[Any]       						= hans_convert_examples_to_features(a_		,   a_		,   a_		,   a_						)

								def gen():
										for ex_index, ex in tqdm.tqdm(enumerate(self.features						)		,   desc="convert examples to features"						):
												if ex_index % 10_000 == 0:
														logger.info("Writing example %d of %d" % (ex_index, len(a_						))						)

												yield (
												    {
												        "example_id": 0,
												        "input_ids": ex.input_ids,
												        "attention_mask": ex.attention_mask,
												        "token_type_ids": ex.token_type_ids,
												    },
												    ex.label,
												)

								lowerCAmelCase  :     int       						= tf.data.Dataset.from_generator(
								    a_		,   (
								        {
								            "example_id": tf.intaa,
								            "input_ids": tf.intaa,
								            "attention_mask": tf.intaa,
								            "token_type_ids": tf.intaa,
								        },
								        tf.intaa,
								    )		,   (
								        {
								            "example_id": tf.TensorShape([]						),
								            "input_ids": tf.TensorShape([None, None]						),
								            "attention_mask": tf.TensorShape([None, None]						),
								            "token_type_ids": tf.TensorShape([None, None]						),
								        },
								        tf.TensorShape([]						),
								    )		,   )



						def 	_lowerCamelCase     ( self						):
								return self.dataset



						def __len__( self						):
								return len(self.features						)



						def __getitem__( self		,   a_						):
								return self.features[i]





						def 	_lowerCamelCase     ( self						):
								return self.label_list
class 		lowerCamelCase	(   _A							):



	def 	_lowerCamelCase     ( self		,   a_						):
			return self._create_examples(self._read_tsv(os.path.join(a_		,   "heuristics_train_set.txt"						)						)		,   "train"						)



	def 	_lowerCamelCase     ( self		,   a_						):
			return self._create_examples(self._read_tsv(os.path.join(a_		,   "heuristics_evaluation_set.txt"						)						)		,   "dev"						)



	def 	_lowerCamelCase     ( self						):
			return ["contradiction", "entailment", "neutral"]





	def 	_lowerCamelCase     ( self		,   a_		,   a_						):
			lowerCAmelCase  :     Optional[int]       						= []
			for i, line in enumerate(a_						):
					if i == 0:
							continue
					lowerCAmelCase  :     Dict       						= "%s-%s" % (set_type, line[0])
					lowerCAmelCase  :     Any       						= line[5]
					lowerCAmelCase  :     List[Any]       						= line[6]
					lowerCAmelCase  :     Union[str, Any]       						= line[7][2:] if line[7].startswith("ex"						) else line[7]
					lowerCAmelCase  :     List[Any]       						= line[0]
					examples.append(InputExample(guid=a_		,   text_a=a_		,   text_b=a_		,   label=a_		,   pairID=a_						)						)
			return examples






def   __A    (				a_    :	List[InputExample]   ,a_    :	List[str]   ,a_    :	int   ,a_    :	PreTrainedTokenizer   ,):
		lowerCAmelCase  :     List[Any]       						= {label: i for i, label in enumerate(a_						)}

		lowerCAmelCase  :     List[str]       						= []
		for ex_index, example in tqdm.tqdm(enumerate(a_						)   ,desc="convert examples to features"						):
				if ex_index % 1_0_0_0_0 == 0:
						logger.info("Writing example %d" % (ex_index)						)

				lowerCAmelCase  :     Any       						= tokenizer(
				    example.text_a   ,example.text_b   ,add_special_tokens=a_   ,max_length=a_   ,padding="max_length"   ,truncation=a_   ,return_overflowing_tokens=a_   ,)

				lowerCAmelCase  :     List[str]       						= label_map[example.label] if example.label in label_map else 0

				lowerCAmelCase  :     Optional[Any]       						= int(example.pairID						)

				features.append(InputFeatures(**a_   ,label=a_   ,pairID=a_						)						)

		for i, example in enumerate(examples[:5]						):
				logger.info("*** Example ***"						)
				logger.info(f'''guid: {example}'''						)
				logger.info(f'''features: {features[i]}'''						)

		return features


lowerCAmelCase									=   {
    """hans""": 3,
}

lowerCAmelCase									=   {
    """hans""": HansProcessor,
}










'''simple docstring'''






import fire

from transformers import AutoConfig, AutoModelForSeqaSeqLM, AutoTokenizer






def   __A    (				a_    :	str   ,a_    :	str   ,**a_    :	int						):
		lowerCAmelCase  :     str       						= AutoConfig.from_pretrained(a_   ,**a_						)
		lowerCAmelCase  :     Any       						= AutoModelForSeqaSeqLM.from_config(a_						)
		model.save_pretrained(a_						)
		AutoTokenizer.from_pretrained(a_						).save_pretrained(a_						)
		return model


if __name__ == "__main__":
					fire.Fire(save_randomly_initialized_version)



