


'''simple docstring'''
from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
		from ..models.auto import AutoTokenizer
class      lowercase			:

		"""simple docstring"""

		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							raise NotImplementedError()




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							raise NotImplementedError()



class      lowercase			(							UpperCamelCase__      ):

		"""simple docstring"""

		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_ = False					,     **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				tokenizer
							UpperCamelCase__     :Optional[Any]							      =				skip_prompt
							UpperCamelCase__     :Optional[Any]							      =				decode_kwargs

							# variables used in the streaming process
							UpperCamelCase__     :Union[str, Any]							      =				[]
							UpperCamelCase__     :int							      =				0
							UpperCamelCase__     :Dict							      =				True




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if len(value.shape							) > 1 and value.shape[0] > 1:
												raise ValueError('''TextStreamer only supports batch size 1'''							)
							elif len(value.shape							) > 1:
												UpperCamelCase__     :int							      =				value[0]

							if self.skip_prompt and self.next_tokens_are_prompt:
												UpperCamelCase__     :str							      =				False
												return

							# Add the new token to the cache and decodes the entire thing.
							self.token_cache.extend(value.tolist()							)
							UpperCamelCase__     :Optional[int]							      =				self.tokenizer.decode(self.token_cache					,     **self.decode_kwargs							)

							# After the symbol for a new line, we flush the cache.
							if text.endswith('''\n'''							):
												UpperCamelCase__     :Dict							      =				text[self.print_len :]
												UpperCamelCase__     :Tuple							      =				[]
												UpperCamelCase__     :Optional[Any]							      =				0
							# If the last token is a CJK character, we print the characters.
							elif len(__A							) > 0 and self._is_chinese_char(ord(text[-1]							)							):
												UpperCamelCase__     :str							      =				text[self.print_len :]
												self.print_len += len(__A							)
							# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
							# which may change with the subsequent token -- there are probably smarter ways to do this!)
							else:
												UpperCamelCase__     :Dict							      =				text[self.print_len : text.rfind(''' '''							) + 1]
												self.print_len += len(__A							)

							self.on_finalized_text(__A							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							if len(self.token_cache							) > 0:
												UpperCamelCase__     :List[Any]							      =				self.tokenizer.decode(self.token_cache					,     **self.decode_kwargs							)
												UpperCamelCase__     :List[Any]							      =				text[self.print_len :]
												UpperCamelCase__     :List[Any]							      =				[]
												UpperCamelCase__     :Dict							      =				0
							else:
												UpperCamelCase__     :Dict							      =				''''''

							UpperCamelCase__     :List[str]							      =				True
							self.on_finalized_text(__A					,     stream_end=__A							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = False							):

							'''simple docstring'''


							print(__A					,     flush=__A					,     end='''''' if not stream_end else None							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							if (
							    (cp >= 0x4_e00 and cp <= 0x9_fff)
							    or (cp >= 0x3_400 and cp <= 0x4_dbf)  #
							    or (cp >= 0x20_000 and cp <= 0x2a_6df)  #
							    or (cp >= 0x2a_700 and cp <= 0x2b_73f)  #
							    or (cp >= 0x2b_740 and cp <= 0x2b_81f)  #
							    or (cp >= 0x2b_820 and cp <= 0x2c_eaf)  #
							    or (cp >= 0xf_900 and cp <= 0xf_aff)
							    or (cp >= 0x2f_800 and cp <= 0x2f_a1f)  #
							):  #
												return True

							return False



class      lowercase			(							UpperCamelCase__      ):

		"""simple docstring"""

		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_ = False					,     UpperCamelCase_ = None					,     **UpperCamelCase_							):

							'''simple docstring'''


							super().__init__(__A					,     __A					,     **__A							)
							UpperCamelCase__     :int							      =				Queue()
							UpperCamelCase__     :Union[str, Any]							      =				None
							UpperCamelCase__     :Dict							      =				timeout




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_ = False							):

							'''simple docstring'''


							self.text_queue.put(__A					,     timeout=self.timeout							)
							if stream_end:
												self.text_queue.put(self.stop_signal					,     timeout=self.timeout							)




		def __iter__(	self							):

							'''simple docstring'''


							return self




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.text_queue.get(timeout=self.timeout							)
							if value == self.stop_signal:
												raise StopIteration()
							else:
												return value



'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							   =  logging.get_logger(__name__)

__snake_case							   =  {
    '''transfo-xl-wt103''': '''https://huggingface.co/transfo-xl-wt103/resolve/main/config.json''',
}
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'transfo-xl'
		_a									=       ['mems']
		_a									=       {
		    'n_token': 'vocab_size',
		    'hidden_size': 'd_model',
		    'num_attention_heads': 'n_head',
		    'num_hidden_layers': 'n_layer',
		}




		def __init__(	self					,     UpperCamelCase_=267735					,     UpperCamelCase_=[20000, 40000, 200000]					,     UpperCamelCase_=1024					,     UpperCamelCase_=1024					,     UpperCamelCase_=16					,     UpperCamelCase_=64					,     UpperCamelCase_=4096					,     UpperCamelCase_=4					,     UpperCamelCase_=False					,     UpperCamelCase_=18					,     UpperCamelCase_=1600					,     UpperCamelCase_=1000					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=0					,     UpperCamelCase_=-1					,     UpperCamelCase_=True					,     UpperCamelCase_=0.1					,     UpperCamelCase_=0.0					,     UpperCamelCase_=True					,     UpperCamelCase_="normal"					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.01					,     UpperCamelCase_=0.02					,     UpperCamelCase_=1e-5					,     UpperCamelCase_=0					,     **UpperCamelCase_					,     ):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				vocab_size
							UpperCamelCase__     :Optional[int]							      =				[]
							self.cutoffs.extend(UpperCamelCase_							)
							if proj_share_all_but_first:
												UpperCamelCase__     :Any							      =				[False] + [True] * len(self.cutoffs							)
							else:
												UpperCamelCase__     :Dict							      =				[False] + [False] * len(self.cutoffs							)
							UpperCamelCase__     :Union[str, Any]							      =				d_model
							UpperCamelCase__     :List[Any]							      =				d_embed
							UpperCamelCase__     :Any							      =				d_head
							UpperCamelCase__     :Tuple							      =				d_inner
							UpperCamelCase__     :int							      =				div_val
							UpperCamelCase__     :int							      =				pre_lnorm
							UpperCamelCase__     :Any							      =				n_layer
							UpperCamelCase__     :int							      =				n_head
							UpperCamelCase__     :Tuple							      =				mem_len
							UpperCamelCase__     :List[str]							      =				same_length
							UpperCamelCase__     :Any							      =				attn_type
							UpperCamelCase__     :List[str]							      =				clamp_len
							UpperCamelCase__     :Union[str, Any]							      =				sample_softmax
							UpperCamelCase__     :List[Any]							      =				adaptive
							UpperCamelCase__     :List[Any]							      =				dropout
							UpperCamelCase__     :Optional[Any]							      =				dropatt
							UpperCamelCase__     :str							      =				untie_r
							UpperCamelCase__     :List[Any]							      =				init
							UpperCamelCase__     :Optional[Any]							      =				init_range
							UpperCamelCase__     :int							      =				proj_init_std
							UpperCamelCase__     :List[str]							      =				init_std
							UpperCamelCase__     :Optional[int]							      =				layer_norm_epsilon
							super().__init__(eos_token_id=UpperCamelCase_					,     **UpperCamelCase_							)




		@property
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							logger.info(F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)
							return -1




		@max_position_embeddings.setter
		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							raise NotImplementedError(
							    F'''The model {self.model_type} is one of the few models that has no sequence length limit.'''							)