




'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


A									=logging.get_logger(__name__)

A									={
    'transfo-xl-wt103': 'https://huggingface.co/transfo-xl-wt103/resolve/main/config.json',
}


class     _a						(			__a				):
       __a       :  List[str]            =      """transfo-xl"""
       __a       :  Tuple            =      ["""mems"""]
       __a       :  Dict            =      {
           """n_token""": """vocab_size""",
           """hidden_size""": """d_model""",
           """num_attention_heads""": """n_head""",
           """num_hidden_layers""": """n_layer""",
       }
       def __init__(			self		:			Optional[int]     ,       lowercase		:			str=267_735     ,       lowercase		:			Optional[Any]=[20_000, 40_000, 200_000]     ,       lowercase		:			Optional[Any]=1_024     ,       lowercase		:			Dict=1_024     ,       lowercase		:			str=16     ,       lowercase		:			List[str]=64     ,       lowercase		:			Tuple=4_096     ,       lowercase		:			Union[str, Any]=4     ,       lowercase		:			Any=False     ,       lowercase		:			str=18     ,       lowercase		:			Tuple=1_600     ,       lowercase		:			int=1_000     ,       lowercase		:			List[Any]=True     ,       lowercase		:			Optional[Any]=True     ,       lowercase		:			Optional[Any]=0     ,       lowercase		:			Optional[int]=-1     ,       lowercase		:			List[Any]=True     ,       lowercase		:			int=0.1     ,       lowercase		:			Any=0.0     ,       lowercase		:			Union[str, Any]=True     ,       lowercase		:			Union[str, Any]="normal"     ,       lowercase		:			Union[str, Any]=0.01     ,       lowercase		:			Dict=0.01     ,       lowercase		:			Optional[int]=0.02     ,       lowercase		:			List[Any]=1E-5     ,       lowercase		:			Any=0     ,       **lowercase		:			int     ,       ):



           '''simple docstring'''

           UpperCAmelCase     =	vocab_size
           UpperCAmelCase     =	[]
           self.cutoffs.extend(lowercase       )
           if proj_share_all_but_first:
               UpperCAmelCase     =	[False] + [True] * len(self.cutoffs       )
           else:
               UpperCAmelCase     =	[False] + [False] * len(self.cutoffs       )
           UpperCAmelCase     =	d_model
           UpperCAmelCase     =	d_embed
           UpperCAmelCase     =	d_head
           UpperCAmelCase     =	d_inner
           UpperCAmelCase     =	div_val
           UpperCAmelCase     =	pre_lnorm
           UpperCAmelCase     =	n_layer
           UpperCAmelCase     =	n_head
           UpperCAmelCase     =	mem_len
           UpperCAmelCase     =	same_length
           UpperCAmelCase     =	attn_type
           UpperCAmelCase     =	clamp_len
           UpperCAmelCase     =	sample_softmax
           UpperCAmelCase     =	adaptive
           UpperCAmelCase     =	dropout
           UpperCAmelCase     =	dropatt
           UpperCAmelCase     =	untie_r
           UpperCAmelCase     =	init
           UpperCAmelCase     =	init_range
           UpperCAmelCase     =	proj_init_std
           UpperCAmelCase     =	init_std
           UpperCAmelCase     =	layer_norm_epsilon
           super().__init__(eos_token_id=lowercase     ,       **lowercase       )
       @property
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit."       )
           return -1







       @max_position_embeddings.setter
       def 					A   (			self		:			int     ,       lowercase		:			Optional[int]       ):



           '''simple docstring'''

           raise NotImplementedError(
               f"The model {self.model_type} is one of the few models that has no sequence length limit."       )










'''simple docstring'''







import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class     _a						(			__a							,							unittest.TestCase				):
       __a       :  Any            =      MgpstrTokenizer
       __a       :  Optional[Any]            =      False
       __a       :  str            =      {}
       __a       :  Optional[int]            =      False
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           super().setUp()

           # fmt: off
           UpperCAmelCase     =	['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
           # fmt: on
           UpperCAmelCase     =	dict(zip(lowercase     ,       range(len(lowercase       )       )       )       )

           UpperCAmelCase     =	os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['''vocab_file''']       )
           with open(self.vocab_file     ,       '''w'''     ,       encoding='''utf-8'''       ) as fp:
               fp.write(json.dumps(lowercase       ) + '''\n'''       )
       def 					A   (			self		:			int     ,       **lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           return MgpstrTokenizer.from_pretrained(self.tmpdirname     ,       **lowercase       )
       def 					A   (			self		:			int     ,       lowercase		:			str       ):



           '''simple docstring'''

           UpperCAmelCase     =	'''tester'''
           UpperCAmelCase     =	'''tester'''
           return input_text, output_text
       @unittest.skip('''MGP-STR always lower cases letters.'''       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           pass
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.get_tokenizers(do_lower_case=lowercase       )
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                   UpperCAmelCase     =	'''[SPECIAL_TOKEN]'''

                   tokenizer.add_special_tokens({'''cls_token''': special_token}       )
                   UpperCAmelCase     =	tokenizer.encode([special_token]     ,       add_special_tokens=lowercase       )
                   self.assertEqual(len(lowercase       )     ,       1       )

                   UpperCAmelCase     =	tokenizer.decode(lowercase     ,       skip_special_tokens=lowercase       )
                   self.assertTrue(special_token not in decoded       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.get_tokenizers()
           for tokenizer in tokenizers:
               with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                   UpperCAmelCase ,  UpperCAmelCase     =	self.get_input_output_texts(lowercase       )

                   UpperCAmelCase     =	tokenizer.tokenize(lowercase       )
                   UpperCAmelCase     =	tokenizer.convert_tokens_to_ids(lowercase       )
                   UpperCAmelCase     =	tokenizer.encode(lowercase     ,       add_special_tokens=lowercase       )
                   self.assertListEqual(lowercase     ,       lowercase       )

                   UpperCAmelCase     =	tokenizer.convert_ids_to_tokens(lowercase       )
                   self.assertNotEqual(len(lowercase       )     ,       0       )
                   UpperCAmelCase     =	tokenizer.decode(lowercase       )
                   self.assertIsInstance(lowercase     ,       lowercase       )

                   self.assertEqual(text_a.replace(''' '''     ,       ''''''       )     ,       lowercase       )
       @unittest.skip('''MGP-STR tokenizer only handles one sequence.'''       )
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           pass







       @unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''       )
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           pass




