



from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__    =	logging.get_logger(__name__)

a__    =	{
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}







class 		snake_case						(	SCREAMING_SNAKE_CASE_  ):



   '''simple docstring'''






   snake_case_					: Union[str, Any]          =		"""transfo-xl"""
   snake_case_					: List[Any]          =		["""mems"""]
   snake_case_					: Optional[int]          =		{
       """n_token""": """vocab_size""",
       """hidden_size""": """d_model""",
       """num_attention_heads""": """n_head""",
       """num_hidden_layers""": """n_layer""",
   }

   def __init__(					self							:			Optional[int]				,		lowerCAmelCase							:			List[Any]=26_7735				,		lowerCAmelCase							:			List[Any]=[2_0000, 4_0000, 20_0000]				,		lowerCAmelCase							:			Dict=1024				,		lowerCAmelCase							:			Any=1024				,		lowerCAmelCase							:			List[Any]=16				,		lowerCAmelCase							:			Optional[Any]=64				,		lowerCAmelCase							:			Any=4096				,		lowerCAmelCase							:			int=4				,		lowerCAmelCase							:			Tuple=False				,		lowerCAmelCase							:			List[str]=18				,		lowerCAmelCase							:			Tuple=1600				,		lowerCAmelCase							:			str=1000				,		lowerCAmelCase							:			Union[str, Any]=True				,		lowerCAmelCase							:			Any=True				,		lowerCAmelCase							:			Union[str, Any]=0				,		lowerCAmelCase							:			int=-1				,		lowerCAmelCase							:			Tuple=True				,		lowerCAmelCase							:			List[str]=0.1				,		lowerCAmelCase							:			List[str]=0.0				,		lowerCAmelCase							:			List[str]=True				,		lowerCAmelCase							:			List[str]="normal"				,		lowerCAmelCase							:			int=0.01				,		lowerCAmelCase							:			List[Any]=0.01				,		lowerCAmelCase							:			List[str]=0.02				,		lowerCAmelCase							:			int=1E-5				,		lowerCAmelCase							:			Optional[int]=0				,		**lowerCAmelCase							:			Optional[int]				,		)				->				int:

    """simple docstring"""







    _snake_case    :  Any					=					vocab_size
    _snake_case    :  List[str]					=					[]
    self.cutoffs.extend(lowerCAmelCase)
    if proj_share_all_but_first:
     _snake_case    :  Dict					=					[False] + [True] * len(self.cutoffs)
    else:
     _snake_case    :  Optional[Any]					=					[False] + [False] * len(self.cutoffs)
    _snake_case    :  Tuple					=					d_model
    _snake_case    :  List[Any]					=					d_embed
    _snake_case    :  Tuple					=					d_head
    _snake_case    :  List[str]					=					d_inner
    _snake_case    :  Optional[Any]					=					div_val
    _snake_case    :  List[str]					=					pre_lnorm
    _snake_case    :  Dict					=					n_layer
    _snake_case    :  str					=					n_head
    _snake_case    :  int					=					mem_len
    _snake_case    :  Union[str, Any]					=					same_length
    _snake_case    :  Tuple					=					attn_type
    _snake_case    :  str					=					clamp_len
    _snake_case    :  Optional[int]					=					sample_softmax
    _snake_case    :  List[Any]					=					adaptive
    _snake_case    :  Tuple					=					dropout
    _snake_case    :  List[Any]					=					dropatt
    _snake_case    :  Optional[Any]					=					untie_r
    _snake_case    :  Optional[int]					=					init
    _snake_case    :  Any					=					init_range
    _snake_case    :  Dict					=					proj_init_std
    _snake_case    :  List[str]					=					init_std
    _snake_case    :  Optional[Any]					=					layer_norm_epsilon
    super().__init__(eos_token_id=lowerCAmelCase				,		**lowerCAmelCase)

   @property
   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.''')
    return -1


   @max_position_embeddings.setter
   def      UpperCamelCase_  (					self							:			Optional[Any]				,		lowerCAmelCase							:			Optional[Any])				->				Union[str, Any]:

    """simple docstring"""







    raise NotImplementedError(
        F'''The model {self.model_type} is one of the few models that has no sequence length limit.''')





import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin







class 		snake_case						(	SCREAMING_SNAKE_CASE_					,unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Optional[Any]          =		CTRLTokenizer
   snake_case_					: Optional[Any]          =		False
   snake_case_					: List[Any]          =		False

   def      UpperCamelCase_  (					self							:			int)				->				Dict:

    """simple docstring"""







    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    _snake_case    :  Dict					=					["""adapt""", """re@@""", """a@@""", """apt""", """c@@""", """t""", """<unk>"""]
    _snake_case    :  Optional[int]					=					dict(zip(lowerCAmelCase				,		range(len(lowerCAmelCase))))
    _snake_case    :  int					=					["""#version: 0.2""", """a p""", """ap t</w>""", """r e""", """a d""", """ad apt</w>""", """"""]
    _snake_case    :  List[Any]					=					{"""unk_token""": """<unk>"""}

    _snake_case    :  Dict					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""vocab_file"""])
    _snake_case    :  Any					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""merges_file"""])
    with open(self.vocab_file				,		"""w"""				,		encoding="""utf-8""") as fp:
     fp.write(json.dumps(lowerCAmelCase) + """\n""")
    with open(self.merges_file				,		"""w"""				,		encoding="""utf-8""") as fp:
     fp.write("""\n""".join(lowerCAmelCase))

   def      UpperCamelCase_  (					self							:			Dict				,		**lowerCAmelCase							:			int)				->				Optional[Any]:

    """simple docstring"""







    kwargs.update(self.special_tokens_map)
    return CTRLTokenizer.from_pretrained(self.tmpdirname				,		**lowerCAmelCase)

   def      UpperCamelCase_  (					self							:			Any				,		lowerCAmelCase							:			Optional[int])				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  Tuple					=					"""adapt react readapt apt"""
    _snake_case    :  Tuple					=					"""adapt react readapt apt"""
    return input_text, output_text


   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				str:

    """simple docstring"""







    _snake_case    :  str					=					CTRLTokenizer(self.vocab_file				,		self.merges_file				,		**self.special_tokens_map)
    _snake_case    :  Dict					=					"""adapt react readapt apt"""
    _snake_case    :  int					=					"""adapt re@@ a@@ c@@ t re@@ adapt apt""".split()
    _snake_case    :  Tuple					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

    _snake_case    :  str					=					tokens + [tokenizer.unk_token]

    _snake_case    :  Union[str, Any]					=					[0, 1, 2, 4, 5, 1, 0, 3, 6]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		lowerCAmelCase)
