


'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
           OpenAIGPTConfig,
           OpenAIGPTDoubleHeadsModel,
           OpenAIGPTForSequenceClassification,
           OpenAIGPTLMHeadModel,
           OpenAIGPTModel,
       )
class 						UpperCamelCase__       :



     """simple docstring"""






     def __init__(			self   ,						snake_case__   ,						snake_case__=13   ,						snake_case__=7   ,						snake_case__=True   ,						snake_case__=True   ,						snake_case__=True   ,						snake_case__=99   ,						snake_case__=32   ,						snake_case__=5   ,						snake_case__=4   ,						snake_case__=37   ,						snake_case__="gelu"   ,						snake_case__=0.1   ,						snake_case__=0.1   ,						snake_case__=512   ,						snake_case__=16   ,						snake_case__=2   ,						snake_case__=0.02   ,						snake_case__=3   ,						snake_case__=4   ,						snake_case__=None   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							parent
            _lowerCAmelCase     :				Union[str, Any]       =							batch_size
            _lowerCAmelCase     :				Tuple       =							seq_length
            _lowerCAmelCase     :				Any       =							is_training
            _lowerCAmelCase     :				Any       =							use_token_type_ids
            _lowerCAmelCase     :				str       =							use_labels
            _lowerCAmelCase     :				int       =							vocab_size
            _lowerCAmelCase     :				Any       =							hidden_size
            _lowerCAmelCase     :				Union[str, Any]       =							num_hidden_layers
            _lowerCAmelCase     :				Optional[int]       =							num_attention_heads
            _lowerCAmelCase     :				Optional[int]       =							intermediate_size
            _lowerCAmelCase     :				int       =							hidden_act
            _lowerCAmelCase     :				Union[str, Any]       =							hidden_dropout_prob
            _lowerCAmelCase     :				int       =							attention_probs_dropout_prob
            _lowerCAmelCase     :				Optional[int]       =							max_position_embeddings
            _lowerCAmelCase     :				Optional[int]       =							type_vocab_size
            _lowerCAmelCase     :				Union[str, Any]       =							type_sequence_label_size
            _lowerCAmelCase     :				Dict       =							initializer_range
            _lowerCAmelCase     :				Tuple       =							num_labels
            _lowerCAmelCase     :				str       =							num_choices
            _lowerCAmelCase     :				Union[str, Any]       =							scope
            _lowerCAmelCase     :				Any       =							self.vocab_size - 1







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							ids_tensor([self.batch_size, self.seq_length]   ,						self.vocab_size )

            _lowerCAmelCase     :				List[str]       =							None
            if self.use_token_type_ids:
                   _lowerCAmelCase     :				Optional[int]       =							ids_tensor([self.batch_size, self.seq_length]   ,						self.type_vocab_size )

            _lowerCAmelCase     :				List[Any]       =							None
            _lowerCAmelCase     :				Tuple       =							None
            _lowerCAmelCase     :				Union[str, Any]       =							None
            if self.use_labels:
                   _lowerCAmelCase     :				Optional[int]       =							ids_tensor([self.batch_size]   ,						self.type_sequence_label_size )
                   _lowerCAmelCase     :				Union[str, Any]       =							ids_tensor([self.batch_size, self.seq_length]   ,						self.num_labels )
                   _lowerCAmelCase     :				Union[str, Any]       =							ids_tensor([self.batch_size]   ,						self.num_choices )

            _lowerCAmelCase     :				Optional[Any]       =							OpenAIGPTConfig(
                vocab_size=self.vocab_size   ,						n_embd=self.hidden_size   ,						n_layer=self.num_hidden_layers   ,						n_head=self.num_attention_heads   ,						n_positions=self.max_position_embeddings   ,						pad_token_id=self.pad_token_id   ,						)

            _lowerCAmelCase     :				str       =							ids_tensor([self.num_hidden_layers, self.num_attention_heads]   ,						2 )

            return (
                config,
                input_ids,
                head_mask,
                token_type_ids,
                sequence_labels,
                token_labels,
                choice_labels,
            )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						*snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							OpenAIGPTModel(config=__UpperCAmelCase )
            model.to(__UpperCAmelCase )
            model.eval()

            _lowerCAmelCase     :				List[Any]       =							model(__UpperCAmelCase   ,						token_type_ids=__UpperCAmelCase   ,						head_mask=__UpperCAmelCase )
            _lowerCAmelCase     :				List[Any]       =							model(__UpperCAmelCase   ,						token_type_ids=__UpperCAmelCase )
            _lowerCAmelCase     :				Optional[Any]       =							model(__UpperCAmelCase )

            self.parent.assertEqual(result.last_hidden_state.shape   ,						(self.batch_size, self.seq_length, self.hidden_size) )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						*snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[Any]       =							OpenAIGPTLMHeadModel(__UpperCAmelCase )
            model.to(__UpperCAmelCase )
            model.eval()

            _lowerCAmelCase     :				List[str]       =							model(__UpperCAmelCase   ,						token_type_ids=__UpperCAmelCase   ,						labels=__UpperCAmelCase )
            self.parent.assertEqual(result.loss.shape   ,						() )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.seq_length, self.vocab_size) )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						*snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Union[str, Any]       =							OpenAIGPTDoubleHeadsModel(__UpperCAmelCase )
            model.to(__UpperCAmelCase )
            model.eval()

            _lowerCAmelCase     :				List[str]       =							model(__UpperCAmelCase   ,						token_type_ids=__UpperCAmelCase   ,						labels=__UpperCAmelCase )
            self.parent.assertEqual(result.loss.shape   ,						() )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.seq_length, self.vocab_size) )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						*snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							self.num_labels
            _lowerCAmelCase     :				Dict       =							OpenAIGPTForSequenceClassification(__UpperCAmelCase )
            model.to(__UpperCAmelCase )
            model.eval()

            _lowerCAmelCase     :				List[Any]       =							ids_tensor([self.batch_size]   ,						self.type_sequence_label_size )
            _lowerCAmelCase     :				Any       =							model(__UpperCAmelCase   ,						token_type_ids=__UpperCAmelCase   ,						labels=__UpperCAmelCase )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.num_labels) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[str]       =							self.prepare_config_and_inputs()
            (
                _lowerCAmelCase   
            )     :				Any       =							config_and_inputs
            _lowerCAmelCase     :				Tuple       =							{
                "input_ids": input_ids,
                "token_type_ids": token_type_ids,
                "head_mask": head_mask,
            }

            return config, inputs_dict
@require_torch
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_     ,       SCREAMING_SNAKE_CASE_     ,       SCREAMING_SNAKE_CASE_     ,       unittest.TestCase						):



     """simple docstring"""






     __magic_name__           =     (
         (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
         if is_torch_available()
         else ()
     )
     __magic_name__           =     (
         (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
     )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
     __magic_name__           =     (
         {
             "feature-extraction": OpenAIGPTModel,
             "text-classification": OpenAIGPTForSequenceClassification,
             "text-generation": OpenAIGPTLMHeadModel,
             "zero-shot": OpenAIGPTForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__ ):

            '''simple docstring'''

            if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                   # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                   # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
                   # tiny config could not be created.
                   return True

            return False







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__=False ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							super()._prepare_for_class(__UpperCAmelCase   ,						__UpperCAmelCase   ,						return_labels=__UpperCAmelCase )

            if return_labels:
                   if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                          _lowerCAmelCase     :				int       =							torch.zeros(
                              (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)   ,						dtype=torch.long   ,						device=__UpperCAmelCase   ,						)
                          _lowerCAmelCase     :				Dict       =							inputs_dict["labels"]
                          _lowerCAmelCase     :				Optional[Any]       =							inputs_dict["labels"]
                          _lowerCAmelCase     :				Any       =							torch.zeros(
                              (self.model_tester.batch_size, self.model_tester.num_choices)   ,						dtype=torch.long   ,						device=__UpperCAmelCase   ,						)
                          _lowerCAmelCase     :				Any       =							torch.zeros(
                              self.model_tester.batch_size   ,						dtype=torch.long   ,						device=__UpperCAmelCase )
            return inputs_dict







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							OpenAIGPTModelTester(self )
            _lowerCAmelCase     :				List[str]       =							ConfigTester(self   ,						config_class=__UpperCAmelCase   ,						n_embd=37 )







     def a						(			self ):

            '''simple docstring'''

            self.config_tester.run_common_tests()







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[str]       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_openai_gpt_model(*__UpperCAmelCase )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_lm_head_model(*__UpperCAmelCase )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_double_lm_head_model(*__UpperCAmelCase )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__UpperCAmelCase )







     @slow
     def a						(			self ):

            '''simple docstring'''

            for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   _lowerCAmelCase     :				int       =							OpenAIGPTModel.from_pretrained(__UpperCAmelCase )
                   self.assertIsNotNone(__UpperCAmelCase )
@require_torch
class 						UpperCamelCase__       (			unittest.TestCase						):



     """simple docstring"""






     @slow
     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							OpenAIGPTLMHeadModel.from_pretrained('openai-gpt' )
            model.to(__UpperCAmelCase )
            _lowerCAmelCase     :				List[Any]       =							torch.tensor([[481, 4735, 544]]   ,						dtype=torch.long   ,						device=__UpperCAmelCase )  # the president is
            _lowerCAmelCase     :				int       =							[
                481,
                4735,
                544,
                246,
                963,
                870,
                762,
                239,
                244,
                4_0477,
                244,
                249,
                719,
                881,
                487,
                544,
                240,
                244,
                603,
                481,
            ]  # the president is a very good man. " \n " i\'m sure he is, " said the

            _lowerCAmelCase     :				List[Any]       =							model.generate(__UpperCAmelCase   ,						do_sample=__UpperCAmelCase )
            self.assertListEqual(output_ids[0].tolist()   ,						__UpperCAmelCase )






import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}






def     SCREAMING_SNAKE_CASE(      __UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase	,				__UpperCamelCase )	->						Dict:
				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys() )}.' )

				if tokenizer_name is None:
								a__	:							str      						= TOKENIZER_CLASSES
				else:
								a__	:							int      						= {tokenizer_name: getattr(__UpperCamelCase	,				tokenizer_name + "Fast" )}

				logger.info(F'Loading tokenizer classes: {tokenizer_names}' )

				for tokenizer_name in tokenizer_names:
								a__	:							Any      						= TOKENIZER_CLASSES[tokenizer_name]

								a__	:							Dict      						= True
								if checkpoint_name is None:
												a__	:							Tuple      						= list(tokenizer_class.max_model_input_sizes.keys() )
								else:
												a__	:							List[str]      						= [checkpoint_name]

								logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}' )

								for checkpoint in checkpoint_names:
												logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}' )

												# Load tokenizer
												a__	:							List[str]      						= tokenizer_class.from_pretrained(__UpperCamelCase	,				force_download=__UpperCamelCase )

												# Save fast tokenizer
												logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}' )

												# For organization names we create sub-directories
												if "/" in checkpoint:
																a__						,   a__	:							Dict      						= checkpoint.split("/" )
																a__	:							Any      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
												elif add_prefix:
																a__	:							int      						= checkpoint
																a__	:							Optional[Any]      						= dump_path
												else:
																a__	:							Dict      						= None
																a__	:							Optional[Any]      						= dump_path

												logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values() )[0]:
																a__	:							Dict      						= list(tokenizer.pretrained_vocab_files_map.values() )[0][checkpoint]
																a__	:							Optional[Any]      						= file_path.split(__UpperCamelCase )[-1][0]
																if next_char == "/":
																				a__	:							Union[str, Any]      						= os.path.join(__UpperCamelCase	,				__UpperCamelCase )
																				a__	:							Any      						= None

																logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}' )

												a__	:							Any      						= tokenizer.save_pretrained(
												    __UpperCamelCase	,				legacy_format=__UpperCamelCase	,				filename_prefix=__UpperCamelCase )
												logger.info(F'=> File names {file_names}' )

												for file_name in file_names:
																if not file_name.endswith("tokenizer.json" ):
																				os.remove(__UpperCamelCase )
																				logger.info(F'=> removing {file_name}' )


if __name__ == "__main__":
						lowerCamelCase		       =      argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
						)
						parser.add_argument(
						    """--tokenizer_name""",
						    default=None,
						    type=str,
						    help=(
						        F'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
						        """download and convert all the checkpoints from AWS."""
						    ),
						)
						parser.add_argument(
						    """--checkpoint_name""",
						    default=None,
						    type=str,
						    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
						)
						parser.add_argument(
						    """--force_download""",
						    action="""store_true""",
						    help="""Re-download checkpoints.""",
						)
						lowerCamelCase		       =      parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)


