





"""simple docstring"""






import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin




class 				__lowercase			(			_UpperCamelCase   ,      unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  CanineTokenizer
					__lowerCAmelCase  =  False







					def  _lowerCamelCase		(						self     ):
									super().setUp()
									__a :						Optional[int]			       =  CanineTokenizer()
									tokenizer.save_pretrained(self.tmpdirname     )







					@cached_property
					def  _lowerCamelCase		(						self     ):
									return CanineTokenizer.from_pretrained('''google/canine-s'''     )







					def  _lowerCamelCase		(						self	,			**_UpperCAmelCase     ):
									__a :						Dict			       =  self.tokenizer_class.from_pretrained(self.tmpdirname	,			**_UpperCAmelCase     )
									__a :						Union[str, Any]			       =  1024
									return tokenizer







					@require_torch
					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  self.canine_tokenizer
									__a :						int			       =  ['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
									# fmt: off
									__a :						Dict			       =  [57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
									# fmt: on
									__a :						Optional[int]			       =  tokenizer(_UpperCAmelCase	,			padding=_UpperCAmelCase	,			return_tensors='''pt'''     )
									self.assertIsInstance(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						str			       =  list(batch.input_ids.numpy()[0]     )

									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									self.assertEqual((2, 39)	,			batch.input_ids.shape     )
									self.assertEqual((2, 39)	,			batch.attention_mask.shape     )







					@require_torch
					def  _lowerCamelCase		(						self     ):
									__a :						int			       =  self.canine_tokenizer
									__a :						Dict			       =  ['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
									__a :						List[str]			       =  tokenizer(_UpperCAmelCase	,			padding=_UpperCAmelCase	,			return_tensors='''pt'''     )
									# check if input_ids, attention_mask and token_type_ids are returned
									self.assertIn('''input_ids'''	,			_UpperCAmelCase     )
									self.assertIn('''attention_mask'''	,			_UpperCAmelCase     )
									self.assertIn('''token_type_ids'''	,			_UpperCAmelCase     )







					@require_torch
					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  self.canine_tokenizer
									__a :						Optional[Any]			       =  [
									    '''What\'s the weater?''',
									    '''It\'s about 25 degrees.''',
									]
									__a :						Optional[int]			       =  tokenizer(
									    text_target=_UpperCAmelCase	,			max_length=32	,			padding='''max_length'''	,			truncation=_UpperCAmelCase	,			return_tensors='''pt'''     )
									self.assertEqual(32	,			targets['''input_ids'''].shape[1]     )







					def  _lowerCamelCase		(						self     ):
									# safety check on max_len default value so we are sure the test works
									__a :						List[Any]			       =  self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	self.assertNotEqual(tokenizer.model_max_length	,			42     )

        # Now let's start the test
									__a :						Dict			       =  self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	__a :						Any			       =  tempfile.mkdtemp()

																	__a :						Union[str, Any]			       =  ''' He is very happy, UNwant\u00E9d,running'''
																	__a :						Optional[int]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	tokenizer.save_pretrained(_UpperCAmelCase     )

																	__a :						int			       =  tokenizer.__class__.from_pretrained(_UpperCAmelCase     )
																	__a :						Any			       =  after_tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

																	shutil.rmtree(_UpperCAmelCase     )

									__a :						Dict			       =  self.get_tokenizers(model_max_length=42     )
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	# Isolate this from the other tests because we save additional tokens/etc
																	__a :						List[str]			       =  tempfile.mkdtemp()

																	__a :						str			       =  ''' He is very happy, UNwant\u00E9d,running'''

																	__a :						Tuple			       =  tokenizer.additional_special_tokens

																	# We can add a new special token for Canine as follows:
																	__a :						Optional[int]			       =  chr(0xe007     )
																	additional_special_tokens.append(_UpperCAmelCase     )
																	tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}     )
																	__a :						List[str]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	tokenizer.save_pretrained(_UpperCAmelCase     )

																	__a :						Optional[int]			       =  tokenizer.__class__.from_pretrained(_UpperCAmelCase     )
																	__a :						List[Any]			       =  after_tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )
																	self.assertIn(_UpperCAmelCase	,			after_tokenizer.additional_special_tokens     )
																	self.assertEqual(after_tokenizer.model_max_length	,			42     )

																	__a :						List[Any]			       =  tokenizer.__class__.from_pretrained(_UpperCAmelCase	,			model_max_length=43     )
																	self.assertEqual(tokenizer.model_max_length	,			43     )

																	shutil.rmtree(_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						List[str]			       =  self.get_tokenizers(do_lower_case=_UpperCAmelCase     )
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	__a       ,						__a :						Union[str, Any]			       =  self.get_clean_sequence(_UpperCAmelCase     )

																	# a special token for Canine can be defined as follows:
																	__a :						Union[str, Any]			       =  0xe005
																	__a :						Any			       =  chr(_UpperCAmelCase     )

																	tokenizer.add_special_tokens({'''cls_token''': special_token}     )
																	__a :						List[str]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	self.assertEqual(len(_UpperCAmelCase     )	,			1     )

																	__a :						int			       =  tokenizer.decode(ids + encoded_special_token	,			clean_up_tokenization_spaces=_UpperCAmelCase     )
																	__a :						Union[str, Any]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )

																	__a :						str			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	__a :						Optional[Any]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	self.assertEqual(_UpperCAmelCase	,			input_encoded + special_token_id     )

																	__a :						Dict			       =  tokenizer.decode(_UpperCAmelCase	,			skip_special_tokens=_UpperCAmelCase     )
																	self.assertTrue(special_token not in decoded     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  self.get_tokenizers(do_lower_case=_UpperCAmelCase     )
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	__a :						Optional[Any]			       =  chr(0xe005     )
																	__a :						Tuple			       =  chr(0xe006     )

																	# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
																	tokenizer.add_tokens([SPECIAL_TOKEN_1]	,			special_tokens=_UpperCAmelCase     )
																	# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
																	# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
																	tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}     )

																	__a :						Dict			       =  tokenizer.tokenize(_UpperCAmelCase     )
																	__a :						Any			       =  tokenizer.tokenize(_UpperCAmelCase     )

																	self.assertEqual(len(_UpperCAmelCase     )	,			1     )
																	self.assertEqual(len(_UpperCAmelCase     )	,			1     )
																	self.assertEqual(token_a[0]	,			_UpperCAmelCase     )
																	self.assertEqual(token_a[0]	,			_UpperCAmelCase     )







					@require_tokenizers
					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  self.get_tokenizers(do_lower_case=_UpperCAmelCase     )
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	# a special token for Canine can be defined as follows:
																	__a :						Optional[Any]			       =  0xe006
																	__a :						List[str]			       =  chr(_UpperCAmelCase     )

																	__a :						Union[str, Any]			       =  AddedToken(_UpperCAmelCase	,			lstrip=_UpperCAmelCase     )
																	tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}     )

																	with tempfile.TemporaryDirectory() as tmp_dir_name:
																					tokenizer.save_pretrained(_UpperCAmelCase     )
																					tokenizer.from_pretrained(_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Dict			       =  []
									if self.test_slow_tokenizer:
													tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

									if self.test_rust_tokenizer:
													tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
													with tempfile.TemporaryDirectory() as tmp_dir:
																	tokenizer_utils.save_pretrained(_UpperCAmelCase     )

																	with open(os.path.join(_UpperCAmelCase	,			'''special_tokens_map.json'''     )	,			encoding='''utf-8'''     ) as json_file:
																					__a :						Any			       =  json.load(_UpperCAmelCase     )

																	with open(os.path.join(_UpperCAmelCase	,			'''tokenizer_config.json'''     )	,			encoding='''utf-8'''     ) as json_file:
																					__a :						Tuple			       =  json.load(_UpperCAmelCase     )

																	# a special token for Canine can be defined as follows:
																	__a :						str			       =  0xe006
																	__a :						int			       =  chr(_UpperCAmelCase     )

																	__a :						Optional[int]			       =  [new_token_a]
																	__a :						Tuple			       =  [new_token_a]

																	with open(os.path.join(_UpperCAmelCase	,			'''special_tokens_map.json'''     )	,			'''w'''	,			encoding='''utf-8'''     ) as outfile:
																					json.dump(_UpperCAmelCase	,			_UpperCAmelCase     )
																	with open(os.path.join(_UpperCAmelCase	,			'''tokenizer_config.json'''     )	,			'''w'''	,			encoding='''utf-8'''     ) as outfile:
																					json.dump(_UpperCAmelCase	,			_UpperCAmelCase     )

																	# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																	# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																	# "special_tokens_map.json" files
																	__a :						Union[str, Any]			       =  tokenizer_class.from_pretrained(_UpperCAmelCase	,			extra_ids=0     )
																	self.assertIn(_UpperCAmelCase	,			tokenizer_without_change_in_init.additional_special_tokens     )
																	# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																	self.assertEqual(
																	    [new_token_a]	,			tokenizer_without_change_in_init.convert_ids_to_tokens(
																	        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]     )     )	,			)

																	__a :						Any			       =  0xe007
																	__a :						Optional[int]			       =  chr(_UpperCAmelCase     )
																	# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																	__a :						Dict			       =  [AddedToken(_UpperCAmelCase	,			lstrip=_UpperCAmelCase     )]
																	__a :						List[Any]			       =  tokenizer_class.from_pretrained(
																	    _UpperCAmelCase	,			additional_special_tokens=_UpperCAmelCase	,			extra_ids=0     )

																	self.assertIn(_UpperCAmelCase	,			tokenizer.additional_special_tokens     )
																	# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
																	self.assertEqual(
																	    [new_token_a]	,			tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]     )     )     )







					@require_tokenizers
					def  _lowerCamelCase		(						self     ):
									__a :						str			       =  self.get_tokenizers(do_lower_case=_UpperCAmelCase     )
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	__a :						List[str]			       =  '''hello world'''
																	if self.space_between_special_tokens:
																					__a :						Any			       =  '''[CLS] hello world [SEP]'''
																	else:
																					__a :						Optional[int]			       =  input
																	__a :						List[Any]			       =  tokenizer.encode(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase     )
																	__a :						Optional[int]			       =  tokenizer.decode(_UpperCAmelCase	,			spaces_between_special_tokens=self.space_between_special_tokens     )
																	self.assertIn(_UpperCAmelCase	,			[output, output.lower()]     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  self.get_tokenizers()
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""     ):
																	__a :						Optional[int]			       =  [
																	    '''bos_token''',
																	    '''eos_token''',
																	    '''unk_token''',
																	    '''sep_token''',
																	    '''pad_token''',
																	    '''cls_token''',
																	    '''mask_token''',
																	]

									__a :						List[str]			       =  '''a'''
									__a :						Optional[Any]			       =  ord(_UpperCAmelCase     )

									for attr in attributes_list:
													setattr(_UpperCAmelCase	,			attr + '''_id'''	,			_UpperCAmelCase     )
													self.assertEqual(getattr(_UpperCAmelCase	,			_UpperCAmelCase     )	,			_UpperCAmelCase     )
													self.assertEqual(getattr(_UpperCAmelCase	,			attr + '''_id'''     )	,			_UpperCAmelCase     )

													setattr(_UpperCAmelCase	,			attr + '''_id'''	,			_UpperCAmelCase     )
													self.assertEqual(getattr(_UpperCAmelCase	,			_UpperCAmelCase     )	,			_UpperCAmelCase     )
													self.assertEqual(getattr(_UpperCAmelCase	,			attr + '''_id'''     )	,			_UpperCAmelCase     )

									setattr(_UpperCAmelCase	,			'''additional_special_tokens_ids'''	,			[]     )
									self.assertListEqual(getattr(_UpperCAmelCase	,			'''additional_special_tokens'''     )	,			[]     )
									self.assertListEqual(getattr(_UpperCAmelCase	,			'''additional_special_tokens_ids'''     )	,			[]     )

									__a :						Optional[Any]			       =  0xe006
									__a :						List[Any]			       =  chr(_UpperCAmelCase     )
									setattr(_UpperCAmelCase	,			'''additional_special_tokens_ids'''	,			[additional_special_token_id]     )
									self.assertListEqual(getattr(_UpperCAmelCase	,			'''additional_special_tokens'''     )	,			[additional_special_token]     )
									self.assertListEqual(getattr(_UpperCAmelCase	,			'''additional_special_tokens_ids'''     )	,			[additional_special_token_id]     )







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass







					def  _lowerCamelCase		(						self     ):
									pass






"""simple docstring"""






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A 			=  logging.get_logger(__name__)

A 			=  {
    '''microsoft/git-base''': '''https://huggingface.co/microsoft/git-base/resolve/main/config.json''',
}




class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  '''git_vision_model'''







					def __init__(						self	,			_UpperCAmelCase=768	,			_UpperCAmelCase=3072	,			_UpperCAmelCase=12	,			_UpperCAmelCase=12	,			_UpperCAmelCase=3	,			_UpperCAmelCase=224	,			_UpperCAmelCase=16	,			_UpperCAmelCase="quick_gelu"	,			_UpperCAmelCase=1e-5	,			_UpperCAmelCase=0.0	,			_UpperCAmelCase=0.0_2	,			**_UpperCAmelCase	,			):
									super().__init__(**_UpperCAmelCase     )

									__a :						Union[str, Any]			       =  hidden_size
									__a :						Optional[int]			       =  intermediate_size
									__a :						Tuple			       =  num_hidden_layers
									__a :						Tuple			       =  num_attention_heads
									__a :						Optional[int]			       =  num_channels
									__a :						Tuple			       =  patch_size
									__a :						Any			       =  image_size
									__a :						Optional[Any]			       =  initializer_range
									__a :						Optional[int]			       =  attention_dropout
									__a :						Any			       =  layer_norm_eps
									__a :						Dict			       =  hidden_act







					@classmethod
					def  _lowerCamelCase		(						cls	,			_UpperCAmelCase	,			**_UpperCAmelCase     ):
									cls._set_token_in_kwargs(_UpperCAmelCase     )

									__a       ,						__a :						Dict			       =  cls.get_config_dict(_UpperCAmelCase	,			**_UpperCAmelCase     )

									# get the vision config dict if we are loading from GITConfig
									if config_dict.get('''model_type'''     ) == "git":
													__a :						int			       =  config_dict['''vision_config''']

									if "model_type" in config_dict and hasattr(cls	,			'''model_type'''     ) and config_dict["model_type"] != cls.model_type:
													logger.warning(
													    f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
													    f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""     )

									return cls.from_dict(_UpperCAmelCase	,			**_UpperCAmelCase     )






class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  '''git'''







					def __init__(						self	,			_UpperCAmelCase=None	,			_UpperCAmelCase=30522	,			_UpperCAmelCase=768	,			_UpperCAmelCase=6	,			_UpperCAmelCase=12	,			_UpperCAmelCase=3072	,			_UpperCAmelCase="gelu"	,			_UpperCAmelCase=0.1	,			_UpperCAmelCase=0.1	,			_UpperCAmelCase=1024	,			_UpperCAmelCase=0.0_2	,			_UpperCAmelCase=1e-1_2	,			_UpperCAmelCase=0	,			_UpperCAmelCase="absolute"	,			_UpperCAmelCase=True	,			_UpperCAmelCase=False	,			_UpperCAmelCase=101	,			_UpperCAmelCase=102	,			_UpperCAmelCase=None	,			**_UpperCAmelCase	,			):
									super().__init__(bos_token_id=_UpperCAmelCase	,			eos_token_id=_UpperCAmelCase	,			pad_token_id=_UpperCAmelCase	,			**_UpperCAmelCase     )

									if vision_config is None:
													__a :						Union[str, Any]			       =  {}
													logger.info('''vision_config is None. initializing the GitVisionConfig with default values.'''     )

									__a :						Union[str, Any]			       =  GitVisionConfig(**_UpperCAmelCase     )
									__a :						List[str]			       =  vocab_size
									__a :						Any			       =  hidden_size
									__a :						Dict			       =  num_hidden_layers
									__a :						Tuple			       =  num_attention_heads
									__a :						List[str]			       =  hidden_act
									__a :						Any			       =  intermediate_size
									__a :						int			       =  hidden_dropout_prob
									__a :						Dict			       =  attention_probs_dropout_prob
									__a :						Tuple			       =  max_position_embeddings
									__a :						int			       =  initializer_range
									__a :						List[str]			       =  layer_norm_eps
									__a :						Tuple			       =  position_embedding_type
									__a :						Optional[int]			       =  use_cache
									__a :						str			       =  tie_word_embeddings
									__a :						Optional[Any]			       =  num_image_with_embedding

									__a :						str			       =  bos_token_id
									__a :						Dict			       =  eos_token_id







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  copy.deepcopy(self.__dict__     )
									__a :						int			       =  self.vision_config.to_dict()
									__a :						Tuple			       =  self.__class__.model_type
									return output