
"""simple docstring"""


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowerCamelCase__					(					__magic_name__ ,	unittest.TestCase				):





			'''simple docstring'''
			lowerCamelCase  					=		PreTrainedTokenizerFast
			lowerCamelCase  					=		False
			lowerCamelCase  					=		True
			lowerCamelCase  					=		'''tokenizer_file'''



			def 					_lowerCAmelCase		(							self  ) ->     Dict:
				_lowerCAmelCase								=False  # because we don't have pretrained_vocab_files_map
				super().setUp()
				_lowerCAmelCase								=True

				_lowerCAmelCase								=["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
				_lowerCAmelCase								="""SaulLu/dummy-tokenizer-bytelevel-bpe"""

				# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
				_lowerCAmelCase								=[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

				_lowerCAmelCase								=PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
				tokenizer.save_pretrained(self.tmpdirname  )



			def 					_lowerCAmelCase		(							self  ) ->     Optional[Any]:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     Any:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     List[str]:
				# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
				# model
				pass



			def 					_lowerCAmelCase		(							self  ) ->     Optional[Any]:
				# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
				pass



			def 					_lowerCAmelCase		(							self  ) ->     str:
				_lowerCAmelCase								=self.tmpdirname
				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						try:
							_lowerCAmelCase								=tempfile.mkdtemp()
							_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase     ,     **__UpperCAmelCase  )

							tokenizer.save_pretrained(self.tmpdirname  )
							super().test_training_new_tokenizer()
						finally:
							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
							# is restored
							shutil.rmtree(self.tmpdirname  )
							_lowerCAmelCase								=tmpdirname_orig



			def 					_lowerCAmelCase		(							self  ) ->     List[str]:
				_lowerCAmelCase								=self.tmpdirname
				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
						try:
							_lowerCAmelCase								=tempfile.mkdtemp()
							_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase     ,     **__UpperCAmelCase  )

							tokenizer.save_pretrained(self.tmpdirname  )
							super().test_training_new_tokenizer_with_special_tokens_change()
						finally:
							# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
							# is restored
							shutil.rmtree(self.tmpdirname  )
							_lowerCAmelCase								=tmpdirname_orig



			def 					_lowerCAmelCase		(							self  ) ->     Tuple:
				_lowerCAmelCase								=self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

				_lowerCAmelCase								=("""a""" for _ in range(10_00  ))
				_lowerCAmelCase								=tokenizer.train_new_from_iterator(text_iterator=__UpperCAmelCase     ,     length=10_00     ,     vocab_size=50  )

				_lowerCAmelCase								=new_tokenizer.encode("""a🤗"""  )
				self.assertEqual(__UpperCAmelCase     ,     [64, 1_72, 2_53, 97, 2_45]  )







@require_tokenizers
class 		lowerCamelCase__					(					unittest.TestCase				):





			'''simple docstring'''



			def 					_lowerCAmelCase		(							self  ) ->     str:
				_lowerCAmelCase								=AutoTokenizer.from_pretrained("""bert-base-cased"""  )
				_lowerCAmelCase								=json.loads(tokenizer._tokenizer.to_str()  )
				_lowerCAmelCase								=len(__UpperCAmelCase  )

				with tempfile.TemporaryDirectory() as tmp_dir:
					# Hack to save this in the tokenizer_config.json
					_lowerCAmelCase								=["""tokenizer.4.0.0.json"""]
					tokenizer.save_pretrained(__UpperCAmelCase  )
					json.dump(__UpperCAmelCase     ,     open(os.path.join(__UpperCAmelCase     ,     """tokenizer.4.0.0.json"""  )     ,     """w"""  )  )

					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
					_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
					self.assertEqual(len(__UpperCAmelCase  )     ,     len(__UpperCAmelCase  ) + 1  )
					_lowerCAmelCase								=json.loads(new_tokenizer._tokenizer.to_str()  )
					self.assertIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )

					# Will need to be adjusted if we reach v42 and this test is still here.
					# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
					shutil.move(os.path.join(__UpperCAmelCase     ,     """tokenizer.4.0.0.json"""  )     ,     os.path.join(__UpperCAmelCase     ,     """tokenizer.42.0.0.json"""  )  )
					_lowerCAmelCase								=["""tokenizer.42.0.0.json"""]
					tokenizer.save_pretrained(__UpperCAmelCase  )
					_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
					self.assertEqual(len(__UpperCAmelCase  )     ,     len(__UpperCAmelCase  )  )
					_lowerCAmelCase								=json.loads(new_tokenizer._tokenizer.to_str()  )
					self.assertNotIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )



			def 					_lowerCAmelCase		(							self  ) ->     Any:
				# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
				_lowerCAmelCase								="""hf-internal-testing/test-two-tokenizers"""

				# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
				_lowerCAmelCase								=AutoTokenizer.from_pretrained(__UpperCAmelCase  )
				self.assertEqual(len(__UpperCAmelCase  )     ,     2_89_97  )
				_lowerCAmelCase								=json.loads(tokenizer._tokenizer.to_str()  )
				self.assertIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )

				# Testing an older version by monkey-patching the version in the module it's used.
				import transformers as old_transformers

				_lowerCAmelCase								="""3.0.0"""
				_lowerCAmelCase								=old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCAmelCase  )
				self.assertEqual(len(__UpperCAmelCase  )     ,     2_89_96  )
				_lowerCAmelCase								=json.loads(old_tokenizer._tokenizer.to_str()  )
				self.assertNotIn("""huggingface"""     ,     json_tokenizer["""model"""]["""vocab"""]  )







@require_tokenizers
class 		lowerCamelCase__					(					unittest.TestCase				):





			'''simple docstring'''



			def 					_lowerCAmelCase		(							self  ) ->     str:
				# See https://github.com/huggingface/transformers/pull/12550
				# and https://github.com/huggingface/tokenizers/issues/537
				_lowerCAmelCase								=PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""  )
				_lowerCAmelCase								="""The Matrix is a 1999 science fiction action film."""

				with concurrent.futures.ThreadPoolExecutor() as executor:
					_lowerCAmelCase								=[executor.submit(self.fetch     ,     __UpperCAmelCase     ,     __UpperCAmelCase  ) for i in range(10  )]
					_lowerCAmelCase								=[future.result() for future in futures]
					self.assertEqual(__UpperCAmelCase     ,     [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase  ) ->     Tuple:
				return tokenizer.encode(__UpperCAmelCase     ,     truncation="""longest_first"""     ,     padding="""longest"""  )







'''simple docstring'''






import inspect
import unittest

from transformers import is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import MODEL_MAPPING, PoolFormerConfig, PoolFormerForImageClassification, PoolFormerModel
		from transformers.models.poolformer.modeling_poolformer import POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from PIL import Image

		from transformers import PoolFormerImageProcessor




class 		a__				(							a__  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Union[str, Any]:
														lowerCAmelCase__          =	self.config_class(**self.inputs_dict   )
														self.parent.assertTrue(hasattr(lowerCamelCase_					,    '''hidden_sizes'''   )   )
														self.parent.assertTrue(hasattr(lowerCamelCase_					,    '''num_encoder_blocks'''   )   )




class 		a__				:





							'''simple docstring'''






							def __init__(		self					,    lowerCamelCase_					,    lowerCamelCase_=13					,    lowerCamelCase_=64					,    lowerCamelCase_=3					,    lowerCamelCase_=4					,    lowerCamelCase_=[2, 2, 2, 2]					,    lowerCamelCase_=[8, 4, 2, 1]					,    lowerCamelCase_=[16, 32, 64, 1_28]					,    lowerCamelCase_=[1, 4, 8, 16]					,    lowerCamelCase_=False					,    lowerCamelCase_=True					,    lowerCamelCase_="gelu"					,    lowerCamelCase_=0.1					,    lowerCamelCase_=0.02					,    lowerCamelCase_=3					,    lowerCamelCase_=None					,    )       ->							Any:
														lowerCAmelCase__          =	parent
														lowerCAmelCase__          =	batch_size
														lowerCAmelCase__          =	image_size
														lowerCAmelCase__          =	num_channels
														lowerCAmelCase__          =	num_encoder_blocks
														lowerCAmelCase__          =	sr_ratios
														lowerCAmelCase__          =	depths
														lowerCAmelCase__          =	hidden_sizes
														lowerCAmelCase__          =	downsampling_rates
														lowerCAmelCase__          =	is_training
														lowerCAmelCase__          =	use_labels
														lowerCAmelCase__          =	hidden_act
														lowerCAmelCase__          =	hidden_dropout_prob
														lowerCAmelCase__          =	initializer_range
														lowerCAmelCase__          =	num_labels
														lowerCAmelCase__          =	scope







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Optional[int]:
														lowerCAmelCase__          =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

														lowerCAmelCase__          =	None
														if self.use_labels:
																					lowerCAmelCase__          =	ids_tensor([self.batch_size, self.image_size, self.image_size]					,    self.num_labels   )

														lowerCAmelCase__          =	PoolFormerConfig(
														    image_size=self.image_size					,    num_channels=self.num_channels					,    num_encoder_blocks=self.num_encoder_blocks					,    depths=self.depths					,    hidden_sizes=self.hidden_sizes					,    hidden_act=self.hidden_act					,    hidden_dropout_prob=self.hidden_dropout_prob					,    initializer_range=self.initializer_range					,    )

														return config, pixel_values, labels







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )       ->							Any:
														lowerCAmelCase__          =	PoolFormerModel(config=lowerCamelCase_   )
														model.to(lowerCamelCase_   )
														model.eval()
														lowerCAmelCase__          =	model(lowerCamelCase_   )
														lowerCAmelCase__          =	lowerCAmelCase__          =	self.image_size // 32.0
														self.parent.assertEqual(
														    result.last_hidden_state.shape					,    (self.batch_size, self.hidden_sizes[-1], expected_height, expected_width)   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							str:
														lowerCAmelCase__          =	self.prepare_config_and_inputs()
														lowerCAmelCase__   ,					lowerCAmelCase__   ,					lowerCAmelCase__          =	config_and_inputs
														lowerCAmelCase__          =	{'''pixel_values''': pixel_values}
														return config, inputs_dict




@require_torch
class 		a__				(							a__				,				a__				,				unittest.TestCase  ):





							'''simple docstring'''






							lowercase__			:   Dict							=      (PoolFormerModel, PoolFormerForImageClassification) if is_torch_available() else ()
							lowercase__			:   Any							=      (
							    {"feature-extraction": PoolFormerModel, "image-classification": PoolFormerForImageClassification}
							    if is_torch_available()
							    else {}
							)

							lowercase__			:   Optional[int]							=      False
							lowercase__			:   Optional[int]							=      False
							lowercase__			:   Union[str, Any]							=      False
							lowercase__			:   List[Any]							=      False
							lowercase__			:   Tuple							=      False







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Union[str, Any]:
														lowerCAmelCase__          =	PoolFormerModelTester(self   )
														lowerCAmelCase__          =	PoolFormerConfigTester(self					,    config_class=lowerCamelCase_   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							List[Any]:
														self.config_tester.run_common_tests()







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							List[Any]:
														lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*lowerCamelCase_   )







							@unittest.skip('''PoolFormer does not use inputs_embeds'''   )
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							str:
														pass







							@unittest.skip('''PoolFormer does not have get_input_embeddings method and get_output_embeddings methods'''   )
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Tuple:
														pass







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														def check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   ):
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					model.to(lowerCamelCase_   )
																					model.eval()

																					with torch.no_grad():
																												lowerCAmelCase__          =	model(**self._prepare_for_class(lowerCamelCase_					,    lowerCamelCase_   )   )

																					lowerCAmelCase__          =	outputs.hidden_states

																					lowerCAmelCase__          =	self.model_tester.num_encoder_blocks
																					self.assertEqual(len(lowerCamelCase_   )					,    lowerCamelCase_   )

																					# verify the first hidden states (first block)
																					self.assertListEqual(
																					    list(hidden_states[0].shape[-3:]   )					,    [
																					        self.model_tester.hidden_sizes[0],
																					        self.model_tester.image_size // 4,
																					        self.model_tester.image_size // 4,
																					    ]					,    )

														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__          =	True
																					check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )

																					# check that output_hidden_states also work using config
																					del inputs_dict["output_hidden_states"]
																					lowerCAmelCase__          =	True

																					check_hidden_states_output(lowerCamelCase_					,    lowerCamelCase_					,    lowerCamelCase_   )







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														if not self.model_tester.is_training:
																					return

														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()
														lowerCAmelCase__          =	True

														for model_class in self.all_model_classes:
																					if model_class in get_values(lowerCamelCase_   ):
																												continue
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					model.to(lowerCamelCase_   )
																					model.train()
																					lowerCAmelCase__          =	self._prepare_for_class(lowerCamelCase_					,    lowerCamelCase_					,    return_labels=lowerCamelCase_   )
																					lowerCAmelCase__          =	model(**lowerCamelCase_   ).loss
																					loss.backward()







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														lowerCAmelCase__   ,					lowerCAmelCase__          =	self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__          =	model_class(lowerCamelCase_   )
																					lowerCAmelCase__          =	inspect.signature(model.forward   )
																					# signature.parameters is an OrderedDict => so arg_names order is deterministic
																					lowerCAmelCase__          =	[*signature.parameters.keys()]

																					lowerCAmelCase__          =	['''pixel_values''']
																					self.assertListEqual(arg_names[:1]					,    lowerCamelCase_   )







							@slow
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							int:
														for model_name in POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					lowerCAmelCase__          =	PoolFormerModel.from_pretrained(lowerCamelCase_   )
																					self.assertIsNotNone(lowerCamelCase_   )


def 					_snake_case   (     )						->    Optional[int]:
							lowerCAmelCase__          =	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''       )
							return image






@require_torch
class 		a__				(							unittest.TestCase  ):





							'''simple docstring'''






							@slow
							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Dict:
														lowerCAmelCase__          =	PoolFormerImageProcessor()
														lowerCAmelCase__          =	PoolFormerForImageClassification.from_pretrained('''sail/poolformer_s12'''   ).to(lowerCamelCase_   )

														lowerCAmelCase__          =	image_processor(images=prepare_img()					,    return_tensors='''pt'''   ).to(lowerCamelCase_   )

														# forward pass
														with torch.no_grad():
																					lowerCAmelCase__          =	model(**lowerCamelCase_   )

														# verify the logits
														lowerCAmelCase__          =	torch.Size((1, 10_00)   )
														self.assertEqual(outputs.logits.shape					,    lowerCamelCase_   )

														lowerCAmelCase__          =	torch.tensor([-0.6_113, 0.1_685, -0.0_492]   ).to(lowerCamelCase_   )
														self.assertTrue(torch.allclose(outputs.logits[0, :3]					,    lowerCamelCase_					,    atol=1e-4   )   )