





'''simple docstring'''






import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


__UpperCamelCase					    =	get_tests_dir("fixtures/dummy_feature_extractor_config.json")
__UpperCamelCase					    =	get_tests_dir("fixtures/vocab.json")
__UpperCamelCase					    =	get_tests_dir("fixtures")




class  _A (					unittest.TestCase		):
		lowercase__:   str				     =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']

		def 	lowercase__     (   self					:   List[Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      Optional[int]							      =     0

		def 	lowercase__     (   self					:   Optional[int]			)					->    Optional[int]:


					"""simple docstring"""

					__snake_case					:      int							      =     AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""			)
					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   str			)					->    Union[str, Any]:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmpdirname:
								__snake_case					:      List[Any]							      =     WavaVecaConfig()
								__snake_case					:      List[Any]							      =     AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""			)

								# save in new folder
								model_config.save_pretrained(__magic_name__			)
								processor.save_pretrained(__magic_name__			)

								__snake_case					:      Optional[int]							      =     AutoProcessor.from_pretrained(__magic_name__			)

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   List[str]			)					->    Dict:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmpdirname:
								# copy relevant files
								copyfile(__magic_name__  ,      os.path.join(__magic_name__  ,      __magic_name__			)			)
								copyfile(__magic_name__  ,      os.path.join(__magic_name__  ,      """vocab.json"""			)			)

								__snake_case					:      Optional[Any]							      =     AutoProcessor.from_pretrained(__magic_name__			)

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Dict			)					->    Union[str, Any]:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmpdirname:
								__snake_case					:      Any							      =     WavaVecaFeatureExtractor()
								__snake_case					:      List[Any]							      =     AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""			)

								__snake_case					:      Union[str, Any]							      =     WavaVecaProcessor(__magic_name__  ,      __magic_name__			)

								# save in new folder
								processor.save_pretrained(__magic_name__			)

								# drop `processor_class` in tokenizer
								with open(os.path.join(__magic_name__  ,      __magic_name__			)  ,      """r"""			) as f:
											__snake_case					:      Optional[Any]							      =     json.load(__magic_name__			)
											config_dict.pop("""processor_class"""			)

								with open(os.path.join(__magic_name__  ,      __magic_name__			)  ,      """w"""			) as f:
											f.write(json.dumps(__magic_name__			)			)

								__snake_case					:      List[str]							      =     AutoProcessor.from_pretrained(__magic_name__			)

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    List[str]:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmpdirname:
								__snake_case					:      Optional[int]							      =     WavaVecaFeatureExtractor()
								__snake_case					:      List[Any]							      =     AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""			)

								__snake_case					:      Optional[Any]							      =     WavaVecaProcessor(__magic_name__  ,      __magic_name__			)

								# save in new folder
								processor.save_pretrained(__magic_name__			)

								# drop `processor_class` in feature extractor
								with open(os.path.join(__magic_name__  ,      __magic_name__			)  ,      """r"""			) as f:
											__snake_case					:      str							      =     json.load(__magic_name__			)
											config_dict.pop("""processor_class"""			)

								with open(os.path.join(__magic_name__  ,      __magic_name__			)  ,      """w"""			) as f:
											f.write(json.dumps(__magic_name__			)			)

								__snake_case					:      Union[str, Any]							      =     AutoProcessor.from_pretrained(__magic_name__			)

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   str			)					->    List[Any]:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmpdirname:
								__snake_case					:      List[Any]							      =     WavaVecaConfig(processor_class="""Wav2Vec2Processor"""			)
								model_config.save_pretrained(__magic_name__			)
								# copy relevant files
								copyfile(__magic_name__  ,      os.path.join(__magic_name__  ,      """vocab.json"""			)			)
								# create emtpy sample processor
								with open(os.path.join(__magic_name__  ,      __magic_name__			)  ,      """w"""			) as f:
											f.write("""{}"""			)

								__snake_case					:      int							      =     AutoProcessor.from_pretrained(__magic_name__			)

					self.assertIsInstance(__magic_name__  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Dict			)					->    List[str]:


					"""simple docstring"""

					with self.assertRaises(__magic_name__			):
								__snake_case					:      List[Any]							      =     AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""			)
					# If remote code is disabled, we can't load this config.
					with self.assertRaises(__magic_name__			):
								__snake_case					:      Union[str, Any]							      =     AutoProcessor.from_pretrained(
								    """hf-internal-testing/test_dynamic_processor"""  ,      trust_remote_code=__magic_name__			)

					__snake_case					:      List[Any]							      =     AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""  ,      trust_remote_code=__magic_name__			)
					self.assertTrue(processor.special_attribute_present			)
					self.assertEqual(processor.__class__.__name__  ,      """NewProcessor"""			)

					__snake_case					:      Tuple							      =     processor.feature_extractor
					self.assertTrue(feature_extractor.special_attribute_present			)
					self.assertEqual(feature_extractor.__class__.__name__  ,      """NewFeatureExtractor"""			)

					__snake_case					:      Any							      =     processor.tokenizer
					self.assertTrue(tokenizer.special_attribute_present			)
					if is_tokenizers_available():
								self.assertEqual(tokenizer.__class__.__name__  ,      """NewTokenizerFast"""			)

								# Test we can also load the slow version
								__snake_case					:      List[str]							      =     AutoProcessor.from_pretrained(
								    """hf-internal-testing/test_dynamic_processor"""  ,      trust_remote_code=__magic_name__  ,      use_fast=__magic_name__			)
								__snake_case					:      Optional[int]							      =     new_processor.tokenizer
								self.assertTrue(new_tokenizer.special_attribute_present			)
								self.assertEqual(new_tokenizer.__class__.__name__  ,      """NewTokenizer"""			)
					else:
								self.assertEqual(tokenizer.__class__.__name__  ,      """NewTokenizer"""			)

		def 	lowercase__     (   self					:   List[str]			)					->    str:


					"""simple docstring"""

					try:
								AutoConfig.register("""custom"""  ,      __magic_name__			)
								AutoFeatureExtractor.register(__magic_name__  ,      __magic_name__			)
								AutoTokenizer.register(__magic_name__  ,      slow_tokenizer_class=__magic_name__			)
								AutoProcessor.register(__magic_name__  ,      __magic_name__			)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(__magic_name__			):
											AutoProcessor.register(__magic_name__  ,      __magic_name__			)

								# Now that the config is registered, it can be used as any other config with the auto-API
								__snake_case					:      Union[str, Any]							      =     CustomFeatureExtractor.from_pretrained(__magic_name__			)

								with tempfile.TemporaryDirectory() as tmp_dir:
											__snake_case					:      str							      =     os.path.join(__magic_name__  ,      """vocab.txt"""			)
											with open(__magic_name__  ,      """w"""  ,      encoding="""utf-8"""			) as vocab_writer:
														vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]			)			)
											__snake_case					:      int							      =     CustomTokenizer(__magic_name__			)

								__snake_case					:      str							      =     CustomProcessor(__magic_name__  ,      __magic_name__			)

								with tempfile.TemporaryDirectory() as tmp_dir:
											processor.save_pretrained(__magic_name__			)
											__snake_case					:      Dict							      =     AutoProcessor.from_pretrained(__magic_name__			)
											self.assertIsInstance(__magic_name__  ,      __magic_name__			)

					finally:
								if "custom" in CONFIG_MAPPING._extra_content:
											del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
											del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
											del TOKENIZER_MAPPING._extra_content[CustomConfig]
								if CustomConfig in PROCESSOR_MAPPING._extra_content:
											del PROCESSOR_MAPPING._extra_content[CustomConfig]

		def 	lowercase__     (   self					:   Any			)					->    List[Any]:


					"""simple docstring"""

					class  _A (					__lowercase		):
							lowercase__:   Optional[Any]				     =   False





					class  _A (					__lowercase		):
							lowercase__:   int				     =   False





					class  _A (					__lowercase		):
							lowercase__:   List[Any]				     =   '''AutoFeatureExtractor'''
							lowercase__:   Tuple				     =   '''AutoTokenizer'''
							lowercase__:   Union[str, Any]				     =   False

					try:
								AutoConfig.register("""custom"""  ,      __magic_name__			)
								AutoFeatureExtractor.register(__magic_name__  ,      __magic_name__			)
								AutoTokenizer.register(__magic_name__  ,      slow_tokenizer_class=__magic_name__			)
								AutoProcessor.register(__magic_name__  ,      __magic_name__			)
								# If remote code is not set, the default is to use local classes.
								__snake_case					:      Any							      =     AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""			)
								self.assertEqual(processor.__class__.__name__  ,      """NewProcessor"""			)
								self.assertFalse(processor.special_attribute_present			)
								self.assertFalse(processor.feature_extractor.special_attribute_present			)
								self.assertFalse(processor.tokenizer.special_attribute_present			)

								# If remote code is disabled, we load the local ones.
								__snake_case					:      Optional[Any]							      =     AutoProcessor.from_pretrained(
								    """hf-internal-testing/test_dynamic_processor"""  ,      trust_remote_code=__magic_name__			)
								self.assertEqual(processor.__class__.__name__  ,      """NewProcessor"""			)
								self.assertFalse(processor.special_attribute_present			)
								self.assertFalse(processor.feature_extractor.special_attribute_present			)
								self.assertFalse(processor.tokenizer.special_attribute_present			)

								# If remote is enabled, we load from the Hub.
								__snake_case					:      List[Any]							      =     AutoProcessor.from_pretrained(
								    """hf-internal-testing/test_dynamic_processor"""  ,      trust_remote_code=__magic_name__			)
								self.assertEqual(processor.__class__.__name__  ,      """NewProcessor"""			)
								self.assertTrue(processor.special_attribute_present			)
								self.assertTrue(processor.feature_extractor.special_attribute_present			)
								self.assertTrue(processor.tokenizer.special_attribute_present			)

					finally:
								if "custom" in CONFIG_MAPPING._extra_content:
											del CONFIG_MAPPING._extra_content["custom"]
								if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
											del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
								if CustomConfig in TOKENIZER_MAPPING._extra_content:
											del TOKENIZER_MAPPING._extra_content[CustomConfig]
								if CustomConfig in PROCESSOR_MAPPING._extra_content:
											del PROCESSOR_MAPPING._extra_content[CustomConfig]

		def 	lowercase__     (   self					:   Optional[Any]			)					->    List[Any]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-bert"""			)
					self.assertEqual(processor.__class__.__name__  ,      """BertTokenizerFast"""			)

		def 	lowercase__     (   self					:   List[str]			)					->    str:


					"""simple docstring"""

					__snake_case					:      int							      =     AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-convnext"""			)
					self.assertEqual(processor.__class__.__name__  ,      """ConvNextImageProcessor"""			)





@is_staging_test
class  _A (					unittest.TestCase		):
		lowercase__:   Dict				     =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']

		@classmethod
		def 	lowercase__     (   cls					:   str			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     TOKEN
					HfFolder.save_token(__magic_name__			)

		@classmethod
		def 	lowercase__     (   cls					:   Dict			)					->    str:


					"""simple docstring"""

					try:
								delete_repo(token=cls._token  ,      repo_id="""test-processor"""			)
					except HTTPError:
								pass

					try:
								delete_repo(token=cls._token  ,      repo_id="""valid_org/test-processor-org"""			)
					except HTTPError:
								pass

					try:
								delete_repo(token=cls._token  ,      repo_id="""test-dynamic-processor"""			)
					except HTTPError:
								pass

		def 	lowercase__     (   self					:   str			)					->    str:


					"""simple docstring"""

					__snake_case					:      int							      =     WavaVecaProcessor.from_pretrained(__magic_name__			)
					with tempfile.TemporaryDirectory() as tmp_dir:
								processor.save_pretrained(
								    os.path.join(__magic_name__  ,      """test-processor"""			)  ,      push_to_hub=__magic_name__  ,      use_auth_token=self._token			)

								__snake_case					:      Union[str, Any]							      =     WavaVecaProcessor.from_pretrained(f'''{USER}/test-processor'''			)
								for k, v in processor.feature_extractor.__dict__.items():
											self.assertEqual(__magic_name__  ,      getattr(new_processor.feature_extractor  ,      __magic_name__			)			)
								self.assertDictEqual(new_processor.tokenizer.get_vocab()  ,      processor.tokenizer.get_vocab()			)

		def 	lowercase__     (   self					:   List[str]			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      List[str]							      =     WavaVecaProcessor.from_pretrained(__magic_name__			)

					with tempfile.TemporaryDirectory() as tmp_dir:
								processor.save_pretrained(
								    os.path.join(__magic_name__  ,      """test-processor-org"""			)  ,      push_to_hub=__magic_name__  ,      use_auth_token=self._token  ,      organization="""valid_org"""  ,      )

								__snake_case					:      Tuple							      =     WavaVecaProcessor.from_pretrained("""valid_org/test-processor-org"""			)
								for k, v in processor.feature_extractor.__dict__.items():
											self.assertEqual(__magic_name__  ,      getattr(new_processor.feature_extractor  ,      __magic_name__			)			)
								self.assertDictEqual(new_processor.tokenizer.get_vocab()  ,      processor.tokenizer.get_vocab()			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Union[str, Any]:


					"""simple docstring"""

					CustomFeatureExtractor.register_for_auto_class()
					CustomTokenizer.register_for_auto_class()
					CustomProcessor.register_for_auto_class()

					__snake_case					:      Optional[int]							      =     CustomFeatureExtractor.from_pretrained(__magic_name__			)

					with tempfile.TemporaryDirectory() as tmp_dir:
								__snake_case					:      Optional[Any]							      =     os.path.join(__magic_name__  ,      """vocab.txt"""			)
								with open(__magic_name__  ,      """w"""  ,      encoding="""utf-8"""			) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]			)			)
								__snake_case					:      Union[str, Any]							      =     CustomTokenizer(__magic_name__			)

					__snake_case					:      Optional[int]							      =     CustomProcessor(__magic_name__  ,      __magic_name__			)

					with tempfile.TemporaryDirectory() as tmp_dir:
								create_repo(f'''{USER}/test-dynamic-processor'''  ,      token=self._token			)
								__snake_case					:      str							      =     Repository(__magic_name__  ,      clone_from=f'''{USER}/test-dynamic-processor'''  ,      token=self._token			)
								processor.save_pretrained(__magic_name__			)

								# This has added the proper auto_map field to the feature extractor config
								self.assertDictEqual(
								    processor.feature_extractor.auto_map  ,      {
								        """AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor""",
								        """AutoProcessor""": """custom_processing.CustomProcessor""",
								    }  ,      )

								# This has added the proper auto_map field to the tokenizer config
								with open(os.path.join(__magic_name__  ,      """tokenizer_config.json"""			)			) as f:
											__snake_case					:      Dict							      =     json.load(__magic_name__			)
								self.assertDictEqual(
								    tokenizer_config["""auto_map"""]  ,      {
								        """AutoTokenizer""": ["""custom_tokenization.CustomTokenizer""", None],
								        """AutoProcessor""": """custom_processing.CustomProcessor""",
								    }  ,      )

								# The code has been copied from fixtures
								self.assertTrue(os.path.isfile(os.path.join(__magic_name__  ,      """custom_feature_extraction.py"""			)			)			)
								self.assertTrue(os.path.isfile(os.path.join(__magic_name__  ,      """custom_tokenization.py"""			)			)			)
								self.assertTrue(os.path.isfile(os.path.join(__magic_name__  ,      """custom_processing.py"""			)			)			)

								repo.push_to_hub()

					__snake_case					:      List[str]							      =     AutoProcessor.from_pretrained(f'''{USER}/test-dynamic-processor'''  ,      trust_remote_code=__magic_name__			)
					# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
					self.assertEqual(new_processor.__class__.__name__  ,      """CustomProcessor"""			)





'''simple docstring'''





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class 		__SCREAMING_SNAKE_CASE							:







				def __init__(       self     :		Dict		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str=13		,					UpperCAmelCase__     :		Tuple=7		,					UpperCAmelCase__     :		Dict=True		,					UpperCAmelCase__     :		Optional[int]=True		,					UpperCAmelCase__     :		Union[str, Any]=True		,					UpperCAmelCase__     :		Tuple=True		,					UpperCAmelCase__     :		Optional[Any]=99		,					UpperCAmelCase__     :		Any=16		,					UpperCAmelCase__     :		Optional[Any]=36		,					UpperCAmelCase__     :		str=6		,					UpperCAmelCase__     :		Any=6		,					UpperCAmelCase__     :		Optional[int]=6		,					UpperCAmelCase__     :		Any=37		,					UpperCAmelCase__     :		List[str]="gelu"		,					UpperCAmelCase__     :		Union[str, Any]=0.1		,					UpperCAmelCase__     :		Optional[int]=0.1		,					UpperCAmelCase__     :		List[str]=512		,					UpperCAmelCase__     :		List[str]=16		,					UpperCAmelCase__     :		Optional[int]=2		,					UpperCAmelCase__     :		Any=0.02		,					UpperCAmelCase__     :		Union[str, Any]=3		,					UpperCAmelCase__     :		List[Any]=4		,					UpperCAmelCase__     :		Tuple=None		,					):
										'''simple docstring'''
										lowercase :							str    		=parent
										lowercase :							int    		=batch_size
										lowercase :							Any    		=seq_length
										lowercase :							int    		=is_training
										lowercase :							str    		=use_input_mask
										lowercase :							int    		=use_token_type_ids
										lowercase :							Dict    		=use_labels
										lowercase :							int    		=vocab_size
										lowercase :							str    		=embedding_size
										lowercase :							Union[str, Any]    		=hidden_size
										lowercase :							Tuple    		=num_hidden_layers
										lowercase :							Any    		=num_hidden_groups
										lowercase :							Union[str, Any]    		=num_attention_heads
										lowercase :							Any    		=intermediate_size
										lowercase :							Tuple    		=hidden_act
										lowercase :							Optional[int]    		=hidden_dropout_prob
										lowercase :							Union[str, Any]    		=attention_probs_dropout_prob
										lowercase :							List[Any]    		=max_position_embeddings
										lowercase :							int    		=type_vocab_size
										lowercase :							int    		=type_sequence_label_size
										lowercase :							Any    		=initializer_range
										lowercase :							List[Any]    		=num_labels
										lowercase :							int    		=num_choices
										lowercase :							Optional[int]    		=scope







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							int    		=ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size   )

										lowercase :							Optional[int]    		=None
										if self.use_input_mask:
																lowercase :							List[str]    		=random_attention_mask([self.batch_size, self.seq_length]   )

										lowercase :							Dict    		=None
										if self.use_token_type_ids:
																lowercase :							List[str]    		=ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size   )

										lowercase :							Tuple    		=None
										lowercase :							Any    		=None
										lowercase :							Dict    		=None
										if self.use_labels:
																lowercase :							int    		=ids_tensor([self.batch_size]		,					self.type_sequence_label_size   )
																lowercase :							str    		=ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels   )
																lowercase :							Optional[Any]    		=ids_tensor([self.batch_size]		,					self.num_choices   )

										lowercase :							Any    		=self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										return AlbertConfig(
										    vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					hidden_act=self.hidden_act		,					hidden_dropout_prob=self.hidden_dropout_prob		,					attention_probs_dropout_prob=self.attention_probs_dropout_prob		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					initializer_range=self.initializer_range		,					num_hidden_groups=self.num_hidden_groups		,					)







				def  lowerCamelCase_	(       self     :		Union[str, Any]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		str   ):
										'''simple docstring'''
										lowercase :							int    		=AlbertModel(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__   )
										lowercase :							Dict    		=model(UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__   )
										lowercase :							int    		=model(UpperCAmelCase__   )
										self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)   )
										self.parent.assertEqual(result.pooler_output.shape		,					(self.batch_size, self.hidden_size)   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Tuple   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertForPreTraining(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__		,					sentence_order_label=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.prediction_logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)   )
										self.parent.assertEqual(result.sop_logits.shape		,					(self.batch_size, config.num_labels)   )







				def  lowerCamelCase_	(       self     :		List[Any]		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		int   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertForMaskedLM(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							str    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)   )







				def  lowerCamelCase_	(       self     :		Any		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							List[str]    		=AlbertForQuestionAnswering(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[str]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					start_positions=UpperCAmelCase__		,					end_positions=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.start_logits.shape		,					(self.batch_size, self.seq_length)   )
										self.parent.assertEqual(result.end_logits.shape		,					(self.batch_size, self.seq_length)   )







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=self.num_labels
										lowercase :							Any    		=AlbertForSequenceClassification(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							Dict    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_labels)   )







				def  lowerCamelCase_	(       self     :		Optional[int]		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Dict   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=self.num_labels
										lowercase :							str    		=AlbertForTokenClassification(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.num_labels)   )







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.num_choices
										lowercase :							List[Any]    		=AlbertForMultipleChoice(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[Any]    		=input_ids.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							int    		=token_type_ids.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							int    		=input_mask.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							List[str]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_choices)   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase :							Union[str, Any]    		=self.prepare_config_and_inputs()
										(
										    (
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	
										) :							Dict    		=config_and_inputs
										lowercase :							Optional[Any]    		={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 		__SCREAMING_SNAKE_CASE							(			lowercase__						,							lowercase__						,							unittest.TestCase      ):
				lowerCamelCase_		       =    (
				    (
				        AlbertModel,
				        AlbertForPreTraining,
				        AlbertForMaskedLM,
				        AlbertForMultipleChoice,
				        AlbertForSequenceClassification,
				        AlbertForTokenClassification,
				        AlbertForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				lowerCamelCase_		       =    (
				    {
				        'feature-extraction': AlbertModel,
				        'fill-mask': AlbertForMaskedLM,
				        'question-answering': AlbertForQuestionAnswering,
				        'text-classification': AlbertForSequenceClassification,
				        'token-classification': AlbertForTokenClassification,
				        'zero-shot': AlbertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowerCamelCase_		       =    True







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int=False   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=super()._prepare_for_class(UpperCAmelCase__		,					UpperCAmelCase__		,					return_labels=UpperCAmelCase__   )

										if return_labels:
																if model_class in get_values(UpperCAmelCase__   ):
																						lowercase :							Any    		=torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)		,					dtype=torch.long		,					device=UpperCAmelCase__   )
																						lowercase :							Any    		=torch.zeros(
																						    self.model_tester.batch_size		,					dtype=torch.long		,					device=UpperCAmelCase__   )
										return inputs_dict







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertModelTester(self   )
										lowercase :							Optional[Any]    		=ConfigTester(self		,					config_class=UpperCAmelCase__		,					hidden_size=37   )







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										self.config_tester.run_common_tests()







				def  lowerCamelCase_	(       self     :		Tuple   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Dict    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Dict   ):
										'''simple docstring'''
										lowercase :							Dict    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		str   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase :							str    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										lowercase :							int    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		List[Any]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																lowercase :							Tuple    		=type
																self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				@slow
				def  lowerCamelCase_	(       self     :		Optional[Any]   ):
										'''simple docstring'''
										for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																lowercase :							str    		=AlbertModel.from_pretrained(UpperCAmelCase__   )
																self.assertIsNotNone(UpperCAmelCase__   )





@require_torch
class 		__SCREAMING_SNAKE_CASE							(			unittest.TestCase      ):







				@slow
				def  lowerCamelCase_	(       self     :		Tuple   ):
										'''simple docstring'''
										lowercase :							int    		=AlbertModel.from_pretrained('''albert-base-v2'''   )
										lowercase :							Optional[int]    		=torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]   )
										lowercase :							Any    		=torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
										with torch.no_grad():
																lowercase :							Any    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__   )[0]
										lowercase :							int    		=torch.Size((1, 11, 768)   )
										self.assertEqual(output.shape		,					UpperCAmelCase__   )
										lowercase :							Union[str, Any]    		=torch.tensor(
										    [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]   )

										self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,					UpperCAmelCase__		,					atol=1E-4   )   )





