



import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowercase_:  Optional[int]     						=					logging.getLogger(__name__)



@dataclass(frozen=__snake_case						)
class 						lowercase__      :

     """simple docstring"""






     __UpperCamelCase    :	Tuple		=       4_2
     __UpperCamelCase    :	Dict		=       4_2
     __UpperCamelCase    :	Any		=       None
     __UpperCamelCase    :	Optional[int]		=       None
     __UpperCamelCase    :	List[str]		=       None



@dataclass(frozen=__snake_case						)
class 						lowercase__      :

     """simple docstring"""






     __UpperCamelCase    :	Optional[int]		=       4_2
     __UpperCamelCase    :	Optional[Any]		=       None
     __UpperCamelCase    :	Tuple		=       None
     __UpperCamelCase    :	Optional[Any]		=       None
     __UpperCamelCase    :	int		=       None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset



 class 						lowercase__      (__snake_case						):

      """simple docstring"""






      __UpperCamelCase    :	str		=       4_2


      def __init__( self :	Tuple   ,			__a :	Optional[int]   ,			__a :	int   ,			__a :	Optional[int]   ,			__a :	Union[str, Any] = None   ,			__a :	Union[str, Any]=False   ,			__a :	Dict = False   ,			):
       snake_case__				:							List[str]      							= hans_processors[task]()

       snake_case__				:							List[Any]      							= os.path.join(
           __a   ,			"""cached_{}_{}_{}_{}""".format(
               """dev""" if evaluate else """train"""   ,			tokenizer.__class__.__name__   ,			str(__a )   ,			__a   ,			)   ,			)
       snake_case__				:							Any      							= processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
        # HACK(label indices are swapped in RoBERTa pretrained model)
        snake_case__				:							Any      							= label_list[2], label_list[1]
       snake_case__				:							Optional[int]      							= label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       snake_case__				:							Any      							= cached_features_file + '.lock'
       with FileLock(__a ):
        if os.path.exists(__a ) and not overwrite_cache:
         logger.info(f'Loading features from cached file {cached_features_file}' )
         snake_case__				:							Any      							= torch.load(__a )
        else:
         logger.info(f'Creating features from dataset file at {data_dir}' )

         snake_case__				:							int      							= (
             processor.get_dev_examples(__a ) if evaluate else processor.get_train_examples(__a )
         )

         logger.info("""Training examples: %s"""   ,			len(__a ) )
         snake_case__				:							List[str]      							= hans_convert_examples_to_features(__a   ,			__a   ,			__a   ,			__a )
         logger.info("""Saving features into cached file %s"""   ,			__a )
         torch.save(self.features   ,			__a )


      def __len__( self :	List[Any] ):
       return len(self.features )


      def __getitem__( self :	str   ,			__a :	List[Any] ):
       return self.features[i]


      def 		lowercase   ( self :	List[str] ):
       return self.label_list


if is_tf_available():
 import tensorflow as tf



 class 						lowercase__      :

      """simple docstring"""






      __UpperCamelCase    :	int		=       4_2


      def __init__( self :	Optional[int]   ,			__a :	Optional[Any]   ,			__a :	Any   ,			__a :	Union[str, Any]   ,			__a :	Union[str, Any] = 1_2_8   ,			__a :	List[str]=False   ,			__a :	List[str] = False   ,			):
       snake_case__				:							str      							= hans_processors[task]()
       snake_case__				:							List[Any]      							= processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
        # HACK(label indices are swapped in RoBERTa pretrained model)
        snake_case__				:							str      							= label_list[2], label_list[1]
       snake_case__				:							int      							= label_list

       snake_case__				:							Any      							= processor.get_dev_examples(__a ) if evaluate else processor.get_train_examples(__a )
       snake_case__				:							str      							= hans_convert_examples_to_features(__a   ,			__a   ,			__a   ,			__a )

       def gen():
        for ex_index, ex in tqdm.tqdm(enumerate(self.features )   ,			desc="""convert examples to features""" ):
         if ex_index % 1_0_0_0_0 == 0:
          logger.info("""Writing example %d of %d""" % (ex_index, len(__a )) )

         yield (
             {
                 "example_id": 0,
                 "input_ids": ex.input_ids,
                 "attention_mask": ex.attention_mask,
                 "token_type_ids": ex.token_type_ids,
             },
             ex.label,
         )

       snake_case__				:							Optional[int]      							= tf.data.Dataset.from_generator(
           __a   ,			(
               {
                   """example_id""": tf.intaa,
                   """input_ids""": tf.intaa,
                   """attention_mask""": tf.intaa,
                   """token_type_ids""": tf.intaa,
               },
               tf.intaa,
           )   ,			(
               {
                   """example_id""": tf.TensorShape([] ),
                   """input_ids""": tf.TensorShape([None, None] ),
                   """attention_mask""": tf.TensorShape([None, None] ),
                   """token_type_ids""": tf.TensorShape([None, None] ),
               },
               tf.TensorShape([] ),
           )   ,			)


      def 		lowercase   ( self :	Any ):
       return self.dataset


      def __len__( self :	List[str] ):
       return len(self.features )


      def __getitem__( self :	List[str]   ,			__a :	Union[str, Any] ):
       return self.features[i]


      def 		lowercase   ( self :	Optional[int] ):
       return self.label_list



class 						lowercase__      (__snake_case						):

     """simple docstring"""






     def 		lowercase   ( self :	Tuple   ,			__a :	List[Any] ):
      return self._create_examples(self._read_tsv(os.path.join(__a   ,			"""heuristics_train_set.txt""" ) )   ,			"""train""" )


     def 		lowercase   ( self :	str   ,			__a :	Optional[Any] ):
      return self._create_examples(self._read_tsv(os.path.join(__a   ,			"""heuristics_evaluation_set.txt""" ) )   ,			"""dev""" )


     def 		lowercase   ( self :	str ):
      return ["contradiction", "entailment", "neutral"]


     def 		lowercase   ( self :	Dict   ,			__a :	int   ,			__a :	Optional[int] ):
      snake_case__				:							str      							= []
      for i, line in enumerate(__a ):
       if i == 0:
        continue
       snake_case__				:							Dict      							= '%s-%s' % (set_type, line[0])
       snake_case__				:							Any      							= line[5]
       snake_case__				:							str      							= line[6]
       snake_case__				:							Tuple      							= line[7][2:] if line[7].startswith("""ex""" ) else line[7]
       snake_case__				:							Any      							= line[0]
       examples.append(InputExample(guid=__a   ,			text_a=__a   ,			text_b=__a   ,			label=__a   ,			pairID=__a ) )
      return examples

def 	_lowercase	(     UpperCAmelCase_						,   UpperCAmelCase_						,   UpperCAmelCase_						,   UpperCAmelCase_						,   ):




 """simple docstring"""




 snake_case__				:							Union[str, Any]      							= {label: i for i, label in enumerate(__lowerCamelCase)}

 snake_case__				:							str      							= []
 for ex_index, example in tqdm.tqdm(enumerate(__lowerCamelCase)						,   desc="""convert examples to features"""):
  if ex_index % 10_000 == 0:
   logger.info("""Writing example %d""" % (ex_index))

  snake_case__				:							Tuple      							= tokenizer(
      example.text_a						,   example.text_b						,   add_special_tokens=__lowerCamelCase						,   max_length=__lowerCamelCase						,   padding="""max_length"""						,   truncation=__lowerCamelCase						,   return_overflowing_tokens=__lowerCamelCase						,   )

  snake_case__				:							int      							= label_map[example.label] if example.label in label_map else 0

  snake_case__				:							Union[str, Any]      							= int(example.pairID)

  features.append(InputFeatures(**__lowerCamelCase						,   label=__lowerCamelCase						,   pairID=__lowerCamelCase))

 for i, example in enumerate(examples[:5]):
  logger.info("""*** Example ***""")
  logger.info(F'guid: {example}')
  logger.info(F'features: {features[i]}')

 return features


lowercase_:  List[str]     						=					{
    'hans': 3,
}
lowercase_:  Tuple     						=					{
    'hans': HansProcessor,
}








'''simple docstring'''






import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible






class    __magic_name__				(						unittest.TestCase							):






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =[
              'safety_checker/pytorch_model.bin',
              'safety_checker/model.safetensors',
              'vae/diffusion_pytorch_model.bin',
              'vae/diffusion_pytorch_model.safetensors',
              'text_encoder/pytorch_model.bin',
              'text_encoder/model.safetensors',
              'unet/diffusion_pytorch_model.bin',
              'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertTrue(is_safetensors_compatible(snake_case))






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =[
              'unet/diffusion_pytorch_model.bin',
              'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertTrue(is_safetensors_compatible(snake_case))






       def      lowerCAmelCase						(     self)		->    Any:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       =[
              'safety_checker/pytorch_model.bin',
              'safety_checker/model.safetensors',
              'vae/diffusion_pytorch_model.bin',
              'vae/diffusion_pytorch_model.safetensors',
              'text_encoder/pytorch_model.bin',
              'text_encoder/model.safetensors',
              'unet/diffusion_pytorch_model.bin',
              # Removed: 'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertFalse(is_safetensors_compatible(snake_case))






       def      lowerCAmelCase						(     self)		->    Dict:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       =[
              'text_encoder/pytorch_model.bin',
              'text_encoder/model.safetensors',
          ]
          self.assertTrue(is_safetensors_compatible(snake_case))






       def      lowerCAmelCase						(     self)		->    Any:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =[
              'safety_checker/pytorch_model.bin',
              'safety_checker/model.safetensors',
              'vae/diffusion_pytorch_model.bin',
              'vae/diffusion_pytorch_model.safetensors',
              'text_encoder/pytorch_model.bin',
              # Removed: 'text_encoder/model.safetensors',
              'unet/diffusion_pytorch_model.bin',
              'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertFalse(is_safetensors_compatible(snake_case))






       def      lowerCAmelCase						(     self)		->    List[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       =[
              'safety_checker/pytorch_model.fp16.bin',
              'safety_checker/model.fp16.safetensors',
              'vae/diffusion_pytorch_model.fp16.bin',
              'vae/diffusion_pytorch_model.fp16.safetensors',
              'text_encoder/pytorch_model.fp16.bin',
              'text_encoder/model.fp16.safetensors',
              'unet/diffusion_pytorch_model.fp16.bin',
              'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          _UpperCAmelCase  :  str				       ='fp16'
          self.assertTrue(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =[
              'unet/diffusion_pytorch_model.fp16.bin',
              'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          _UpperCAmelCase  :  int				       ='fp16'
          self.assertTrue(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          # pass variant but use the non-variant filenames
          _UpperCAmelCase  :  Optional[int]				       =[
              'unet/diffusion_pytorch_model.bin',
              'unet/diffusion_pytorch_model.safetensors',
          ]
          _UpperCAmelCase  :  Optional[Any]				       ='fp16'
          self.assertTrue(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  str				       =[
              'safety_checker/pytorch_model.fp16.bin',
              'safety_checker/model.fp16.safetensors',
              'vae/diffusion_pytorch_model.fp16.bin',
              'vae/diffusion_pytorch_model.fp16.safetensors',
              'text_encoder/pytorch_model.fp16.bin',
              'text_encoder/model.fp16.safetensors',
              'unet/diffusion_pytorch_model.fp16.bin',
              # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          _UpperCAmelCase  :  List[str]				       ='fp16'
          self.assertFalse(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Dict				       =[
              'text_encoder/pytorch_model.fp16.bin',
              'text_encoder/model.fp16.safetensors',
          ]
          _UpperCAmelCase  :  Optional[Any]				       ='fp16'
          self.assertTrue(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    Any:




          '''simple docstring'''
          # pass variant but use the non-variant filenames
          _UpperCAmelCase  :  Optional[int]				       =[
              'text_encoder/pytorch_model.bin',
              'text_encoder/model.safetensors',
          ]
          _UpperCAmelCase  :  List[Any]				       ='fp16'
          self.assertTrue(is_safetensors_compatible(snake_case    ,				variant=snake_case))






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  str				       =[
              'safety_checker/pytorch_model.fp16.bin',
              'safety_checker/model.fp16.safetensors',
              'vae/diffusion_pytorch_model.fp16.bin',
              'vae/diffusion_pytorch_model.fp16.safetensors',
              'text_encoder/pytorch_model.fp16.bin',
              # 'text_encoder/model.fp16.safetensors',
              'unet/diffusion_pytorch_model.fp16.bin',
              'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          _UpperCAmelCase  :  Tuple				       ='fp16'
          self.assertFalse(is_safetensors_compatible(snake_case    ,				variant=snake_case))

