


'''simple docstring'''
import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      lowercase			(							__lowerCAmelCase			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       MgpstrTokenizer
		_a									=       False
		_a									=       {}
		_a									=       False




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							# fmt: off
							UpperCamelCase__     :Any							      =				["""[GO]""", """[s]""", """0""", """1""", """2""", """3""", """4""", """5""", """6""", """7""", """8""", """9""", """a""", """b""", """c""", """d""", """e""", """f""", """g""", """h""", """i""", """j""", """k""", """l""", """m""", """n""", """o""", """p""", """q""", """r""", """s""", """t""", """u""", """v""", """w""", """x""", """y""", """z"""]
							# fmt: on
							UpperCamelCase__     :Optional[int]							      =				dict(zip(_UpperCamelCase					,     range(len(_UpperCamelCase							)							)							)							)

							UpperCamelCase__     :str							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as fp:
												fp.write(json.dumps(_UpperCamelCase							) + '''\n'''							)




		def 					lowerCAmelCase__       (	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							return MgpstrTokenizer.from_pretrained(self.tmpdirname					,     **_UpperCamelCase							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				"""tester"""
							UpperCamelCase__     :int							      =				"""tester"""
							return input_text, output_text




		@unittest.skip('''MGP-STR always lower cases letters.'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				self.get_tokenizers(do_lower_case=_UpperCamelCase							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[Any]							      =				"""[SPECIAL_TOKEN]"""

																	tokenizer.add_special_tokens({'''cls_token''': special_token}							)
																	UpperCamelCase__     :Optional[Any]							      =				tokenizer.encode([special_token]					,     add_special_tokens=_UpperCamelCase							)
																	self.assertEqual(len(_UpperCamelCase							)					,     1							)

																	UpperCamelCase__     :Any							      =				tokenizer.decode(_UpperCamelCase					,     skip_special_tokens=_UpperCamelCase							)
																	self.assertTrue(special_token not in decoded							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :List[Any]							      =				self.get_input_output_texts(_UpperCamelCase							)

																	UpperCamelCase__     :Dict							      =				tokenizer.tokenize(_UpperCamelCase							)
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.convert_tokens_to_ids(_UpperCamelCase							)
																	UpperCamelCase__     :Any							      =				tokenizer.encode(_UpperCamelCase					,     add_special_tokens=_UpperCamelCase							)
																	self.assertListEqual(_UpperCamelCase					,     _UpperCamelCase							)

																	UpperCamelCase__     :Optional[Any]							      =				tokenizer.convert_ids_to_tokens(_UpperCamelCase							)
																	self.assertNotEqual(len(_UpperCamelCase							)					,     0							)
																	UpperCamelCase__     :str							      =				tokenizer.decode(_UpperCamelCase							)
																	self.assertIsInstance(_UpperCamelCase					,     _UpperCamelCase							)

																	self.assertEqual(text_a.replace(''' '''					,     ''''''							)					,     _UpperCamelCase							)




		@unittest.skip('''MGP-STR tokenizer only handles one sequence.'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass




		@unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass



'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'transfo-xl'
		_a									=       ['mems']
		_a									=       {
		    'n_token': 'vocab_size',
		    'hidden_size': 'd_model',
		    'num_attention_heads': 'n_head',
		    'num_hidden_layers': 'n_layer',
		}




		def __init__(	self					,     UpperCamelCase_=267735					,     UpperCamelCase_=[20000, 40000, 200000]					,     UpperCamelCase_=1024					,     UpperCamelCase_=1024					,     UpperCamelCase_=16					,     UpperCamelCase_=64					,     UpperCamelCase_=4096					,     UpperCamelCase_=4					,     UpperCamelCase_=False					,     UpperCamelCase_=18					,     UpperCamelCase_=1600					,     UpperCamelCase_=1000					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=0					,     UpperCamelCase_=-1					,     UpperCamelCase_=True					,     UpperCamelCase_=0.1					,     UpperCamelCase_=0.0					,     UpperCamelCase_=True					,     UpperCamelCase_="normal"					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.02					,     UpperCamelCase_=1e-5					,     UpperCamelCase_=0					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				vocab_size
							UpperCamelCase__     :Optional[int]							      =				[]
							self.cutoffs.extend(UpperCamelCase_							)
							if proj_share_all_but_first:
												UpperCamelCase__     :Any							      =				[False] + [True] * len(self.cutoffs							)
							else:
												UpperCamelCase__     :Dict							      =				[False] + [False] * len(self.cutoffs							)
							UpperCamelCase__     :Union[str, Any]							      =				d_model
							UpperCamelCase__     :List[Any]							      =				d_embed
							UpperCamelCase__     :Any							      =				d_head
							UpperCamelCase__     :Tuple							      =				d_inner
							UpperCamelCase__     :int							      =				div_val
							UpperCamelCase__     :int							      =				pre_lnorm
							UpperCamelCase__     :Any							      =				n_layer
							UpperCamelCase__     :int							      =				n_head
							UpperCamelCase__     :Tuple							      =				mem_len
							UpperCamelCase__     :List[str]							      =				same_length
							UpperCamelCase__     :Any							      =				attn_type
							UpperCamelCase__     :List[str]							      =				clamp_len
							UpperCamelCase__     :Union[str, Any]							      =				sample_softmax
							UpperCamelCase__     :List[Any]							      =				adaptive
							UpperCamelCase__     :List[Any]							      =				dropout
							UpperCamelCase__     :Optional[Any]							      =				dropatt
							UpperCamelCase__     :str							      =				untie_r
							UpperCamelCase__     :List[Any]							      =				init
							UpperCamelCase__     :Optional[Any]							      =				init_range
							UpperCamelCase__     :int							      =				proj_init_std
							UpperCamelCase__     :List[str]							      =				init_std
							UpperCamelCase__     :Optional[int]							      =				layer_norm_epsilon
							super().__init__(eos_token_id=UpperCamelCase_					,     **UpperCamelCase_							)




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)
							return -1




		@max_position_embeddings.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)