





"""simple docstring"""





import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__   =  logging.getLogger(__name__)

@dataclass(frozen=_UpperCAmelCase   )
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None







@dataclass(frozen=_UpperCAmelCase   )
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None


if is_torch_available():
       import torch
       from torch.utils.data import Dataset

       class 						lowercase   (   _UpperCAmelCase   ):
             _SCREAMING_SNAKE_CASE							      = 42





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase = None ,		lowercase=False ,		lowercase = False ,		)						->	Optional[Any]:
                   lowerCAmelCase    		=			hans_processors[task]()

                   lowerCAmelCase    		=			os.path.join(
                       lowercase ,		"""cached_{}_{}_{}_{}""".format(
                           """dev""" if evaluate else """train""" ,		tokenizer.__class__.__name__ ,		str(lowercase					) ,		lowercase ,		) ,		)
                   lowerCAmelCase    		=			processor.get_labels()
                   if tokenizer.__class__ in (
                       RobertaTokenizer,
                       RobertaTokenizerFast,
                       XLMRobertaTokenizer,
                       BartTokenizer,
                       BartTokenizerFast,
                   ):
                         # HACK(label indices are swapped in RoBERTa pretrained model)
                         lowerCAmelCase		,						lowerCAmelCase    		=			label_list[2], label_list[1]
                   lowerCAmelCase    		=			label_list

                   # Make sure only the first process in distributed training processes the dataset,
                   # and the others will use the cache.
                   lowerCAmelCase    		=			cached_features_file + """.lock"""
                   with FileLock(lowercase					):
                         if os.path.exists(lowercase					) and not overwrite_cache:
                               logger.info(f'Loading features from cached file {cached_features_file}'					)
                               lowerCAmelCase    		=			torch.load(lowercase					)
                         else:
                               logger.info(f'Creating features from dataset file at {data_dir}'					)

                               lowerCAmelCase    		=			(
                                   processor.get_dev_examples(lowercase					) if evaluate else processor.get_train_examples(lowercase					)
                               )

                               logger.info("""Training examples: %s""" ,		len(lowercase					)					)
                               lowerCAmelCase    		=			hans_convert_examples_to_features(lowercase ,		lowercase ,		lowercase ,		lowercase					)
                               logger.info("""Saving features into cached file %s""" ,		lowercase					)
                               torch.save(self.features ,		lowercase					)





             def __len__(			self					)						->	int:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]





             def        _snake_case     (			self					)						->	Union[str, Any]:
                   return self.label_list


if is_tf_available():
       import tensorflow as tf

       class 						lowercase   :
             _SCREAMING_SNAKE_CASE							      = 42





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase = 128 ,		lowercase=False ,		lowercase = False ,		)						->	Tuple:
                   lowerCAmelCase    		=			hans_processors[task]()
                   lowerCAmelCase    		=			processor.get_labels()
                   if tokenizer.__class__ in (
                       RobertaTokenizer,
                       RobertaTokenizerFast,
                       XLMRobertaTokenizer,
                       BartTokenizer,
                       BartTokenizerFast,
                   ):
                         # HACK(label indices are swapped in RoBERTa pretrained model)
                         lowerCAmelCase		,						lowerCAmelCase    		=			label_list[2], label_list[1]
                   lowerCAmelCase    		=			label_list

                   lowerCAmelCase    		=			processor.get_dev_examples(lowercase					) if evaluate else processor.get_train_examples(lowercase					)
                   lowerCAmelCase    		=			hans_convert_examples_to_features(lowercase ,		lowercase ,		lowercase ,		lowercase					)

                   def gen():
                         for ex_index, ex in tqdm.tqdm(enumerate(self.features					) ,		desc="""convert examples to features"""					):
                               if ex_index % 10_000 == 0:
                                     logger.info("""Writing example %d of %d""" % (ex_index, len(lowercase					))					)

                               yield (
                                   {
                                       "example_id": 0,
                                       "input_ids": ex.input_ids,
                                       "attention_mask": ex.attention_mask,
                                       "token_type_ids": ex.token_type_ids,
                                   },
                                   ex.label,
                               )

                   lowerCAmelCase    		=			tf.data.Dataset.from_generator(
                       lowercase ,		(
                           {
                               """example_id""": tf.intaa,
                               """input_ids""": tf.intaa,
                               """attention_mask""": tf.intaa,
                               """token_type_ids""": tf.intaa,
                           },
                           tf.intaa,
                       ) ,		(
                           {
                               """example_id""": tf.TensorShape([]					),
                               """input_ids""": tf.TensorShape([None, None]					),
                               """attention_mask""": tf.TensorShape([None, None]					),
                               """token_type_ids""": tf.TensorShape([None, None]					),
                           },
                           tf.TensorShape([]					),
                       ) ,		)





             def        _snake_case     (			self					)						->	List[str]:
                   return self.dataset





             def __len__(			self					)						->	str:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]





             def        _snake_case     (			self					)						->	str:
                   return self.label_list

class 						lowercase   (   _UpperCAmelCase   ):





      def        _snake_case     (			self ,		lowercase					)						->	List[str]:
            return self._create_examples(self._read_tsv(os.path.join(lowercase ,		"""heuristics_train_set.txt"""					)					) ,		"""train"""					)





      def        _snake_case     (			self ,		lowercase					)						->	Dict:
            return self._create_examples(self._read_tsv(os.path.join(lowercase ,		"""heuristics_evaluation_set.txt"""					)					) ,		"""dev"""					)





      def        _snake_case     (			self					)						->	Union[str, Any]:
            return ["contradiction", "entailment", "neutral"]





      def        _snake_case     (			self ,		lowercase ,		lowercase					)						->	int:
            lowerCAmelCase    		=			[]
            for i, line in enumerate(lowercase					):
                  if i == 0:
                        continue
                  lowerCAmelCase    		=			"""%s-%s""" % (set_type, line[0])
                  lowerCAmelCase    		=			line[5]
                  lowerCAmelCase    		=			line[6]
                  lowerCAmelCase    		=			line[7][2:] if line[7].startswith("""ex"""					) else line[7]
                  lowerCAmelCase    		=			line[0]
                  examples.append(InputExample(guid=lowercase ,		text_a=lowercase ,		text_b=lowercase ,		label=lowercase ,		pairID=lowercase					)					)
            return examples







def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		List[InputExample]				,		SCREAMING_SNAKE_CASE	:		List[str]				,		SCREAMING_SNAKE_CASE	:		int				,		SCREAMING_SNAKE_CASE	:		PreTrainedTokenizer				,		):







      '''simple docstring'''




      lowerCAmelCase    		=			{label: i for i, label in enumerate(SCREAMING_SNAKE_CASE     )}

      lowerCAmelCase    		=			[]
      for ex_index, example in tqdm.tqdm(enumerate(SCREAMING_SNAKE_CASE     )				,		desc="""convert examples to features"""     ):
            if ex_index % 1_00_00 == 0:
                  logger.info("""Writing example %d""" % (ex_index)     )

            lowerCAmelCase    		=			tokenizer(
                example.text_a				,		example.text_b				,		add_special_tokens=SCREAMING_SNAKE_CASE				,		max_length=SCREAMING_SNAKE_CASE				,		padding="""max_length"""				,		truncation=SCREAMING_SNAKE_CASE				,		return_overflowing_tokens=SCREAMING_SNAKE_CASE				,		)

            lowerCAmelCase    		=			label_map[example.label] if example.label in label_map else 0

            lowerCAmelCase    		=			int(example.pairID     )

            features.append(InputFeatures(**SCREAMING_SNAKE_CASE				,		label=SCREAMING_SNAKE_CASE				,		pairID=SCREAMING_SNAKE_CASE     )     )

      for i, example in enumerate(examples[:5]     ):
            logger.info("""*** Example ***"""     )
            logger.info(F'guid: {example}'     )
            logger.info(F'features: {features[i]}'     )

      return features


SCREAMING_SNAKE_CASE__   =  {
    "hans": 3,
}






SCREAMING_SNAKE_CASE__   =  {
    "hans": HansProcessor,
}













"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE__   =  {
    "configuration_roformer": ["ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "RoFormerConfig", "RoFormerOnnxConfig"],
    "tokenization_roformer": ["RoFormerTokenizer"],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__   =  ["RoFormerTokenizerFast"]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__   =  [
           "ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
           "RoFormerForCausalLM",
           "RoFormerForMaskedLM",
           "RoFormerForMultipleChoice",
           "RoFormerForQuestionAnswering",
           "RoFormerForSequenceClassification",
           "RoFormerForTokenClassification",
           "RoFormerLayer",
           "RoFormerModel",
           "RoFormerPreTrainedModel",
           "load_tf_weights_in_roformer",
       ]


try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__   =  [
           "TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
           "TFRoFormerForCausalLM",
           "TFRoFormerForMaskedLM",
           "TFRoFormerForMultipleChoice",
           "TFRoFormerForQuestionAnswering",
           "TFRoFormerForSequenceClassification",
           "TFRoFormerForTokenClassification",
           "TFRoFormerLayer",
           "TFRoFormerModel",
           "TFRoFormerPreTrainedModel",
       ]


try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       SCREAMING_SNAKE_CASE__   =  [
           "FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
           "FlaxRoFormerForMaskedLM",
           "FlaxRoFormerForMultipleChoice",
           "FlaxRoFormerForQuestionAnswering",
           "FlaxRoFormerForSequenceClassification",
           "FlaxRoFormerForTokenClassification",
           "FlaxRoFormerModel",
           "FlaxRoFormerPreTrainedModel",
       ]


if TYPE_CHECKING:
       from .configuration_roformer import ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RoFormerConfig, RoFormerOnnxConfig
       from .tokenization_roformer import RoFormerTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_roformer_fast import RoFormerTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_roformer import (
                  ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  RoFormerForCausalLM,
                  RoFormerForMaskedLM,
                  RoFormerForMultipleChoice,
                  RoFormerForQuestionAnswering,
                  RoFormerForSequenceClassification,
                  RoFormerForTokenClassification,
                  RoFormerLayer,
                  RoFormerModel,
                  RoFormerPreTrainedModel,
                  load_tf_weights_in_roformer,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_roformer import (
                  TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFRoFormerForCausalLM,
                  TFRoFormerForMaskedLM,
                  TFRoFormerForMultipleChoice,
                  TFRoFormerForQuestionAnswering,
                  TFRoFormerForSequenceClassification,
                  TFRoFormerForTokenClassification,
                  TFRoFormerLayer,
                  TFRoFormerModel,
                  TFRoFormerPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_roformer import (
                  FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  FlaxRoFormerForMaskedLM,
                  FlaxRoFormerForMultipleChoice,
                  FlaxRoFormerForQuestionAnswering,
                  FlaxRoFormerForSequenceClassification,
                  FlaxRoFormerForTokenClassification,
                  FlaxRoFormerModel,
                  FlaxRoFormerPreTrainedModel,
              )


else:
       import sys

       SCREAMING_SNAKE_CASE__   =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






