



"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase     =							{
    """configuration_mega""": ["""MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegaConfig""", """MegaOnnxConfig"""],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCamelCase     =							[
			    """MEGA_PRETRAINED_MODEL_ARCHIVE_LIST""",
			    """MegaForCausalLM""",
			    """MegaForMaskedLM""",
			    """MegaForMultipleChoice""",
			    """MegaForQuestionAnswering""",
			    """MegaForSequenceClassification""",
			    """MegaForTokenClassification""",
			    """MegaModel""",
			    """MegaPreTrainedModel""",
			]

if TYPE_CHECKING:
			from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_mega import (
						    MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
						    MegaForCausalLM,
						    MegaForMaskedLM,
						    MegaForMultipleChoice,
						    MegaForQuestionAnswering,
						    MegaForSequenceClassification,
						    MegaForTokenClassification,
						    MegaModel,
						    MegaPreTrainedModel,
						)

else:
			import sys

			UpperCamelCase     =							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
		from ..models.auto import AutoTokenizer




class 					lowerCamelCase__				:




	'''simple docstring'''




	def   _lowerCamelCase							(   self						:Optional[Any]	,  a						:int		)						->					Any:
								raise NotImplementedError()







	def   _lowerCamelCase							(   self						:Any		)						->					Optional[int]:
								raise NotImplementedError()




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	def __init__(   self						:List[Any]	,  a						:"AutoTokenizer"	,  a						:bool = False	,  **a						:Tuple		)						->					Union[str, Any]:
								__UpperCamelCase      :  int       =   tokenizer
								__UpperCamelCase      :  Any       =   skip_prompt
								__UpperCamelCase      :  Union[str, Any]       =   decode_kwargs

								# variables used in the streaming process
								__UpperCamelCase      :  Union[str, Any]       =   []
								__UpperCamelCase      :  str       =   0
								__UpperCamelCase      :  str       =   True







	def   _lowerCamelCase							(   self						:List[str]	,  a						:Union[str, Any]		)						->					Any:
								if len(value.shape		) > 1 and value.shape[0] > 1:
															raise ValueError("TextStreamer only supports batch size 1"		)
								elif len(value.shape		) > 1:
															__UpperCamelCase      :  Optional[int]       =   value[0]

								if self.skip_prompt and self.next_tokens_are_prompt:
															__UpperCamelCase      :  Optional[Any]       =   False
															return

								# Add the new token to the cache and decodes the entire thing.
								self.token_cache.extend(value.tolist()		)
								__UpperCamelCase      :  Optional[int]       =   self.tokenizer.decode(self.token_cache	,  **self.decode_kwargs		)

								# After the symbol for a new line, we flush the cache.
								if text.endswith("\n"		):
															__UpperCamelCase      :  Optional[int]       =   text[self.print_len :]
															__UpperCamelCase      :  Dict       =   []
															__UpperCamelCase      :  int       =   0
								# If the last token is a CJK character, we print the characters.
								elif len(a		) > 0 and self._is_chinese_char(ord(text[-1]		)		):
															__UpperCamelCase      :  Optional[int]       =   text[self.print_len :]
															self.print_len += len(a		)
								# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
								# which may change with the subsequent token -- there are probably smarter ways to do this!)
								else:
															__UpperCamelCase      :  Any       =   text[self.print_len : text.rfind(" "		) + 1]
															self.print_len += len(a		)

								self.on_finalized_text(a		)







	def   _lowerCamelCase							(   self						:List[str]		)						->					List[str]:
								# Flush the cache, if it exists
								if len(self.token_cache		) > 0:
															__UpperCamelCase      :  str       =   self.tokenizer.decode(self.token_cache	,  **self.decode_kwargs		)
															__UpperCamelCase      :  str       =   text[self.print_len :]
															__UpperCamelCase      :  str       =   []
															__UpperCamelCase      :  int       =   0
								else:
															__UpperCamelCase      :  int       =   ""

								__UpperCamelCase      :  Optional[Any]       =   True
								self.on_finalized_text(a	,  stream_end=a		)







	def   _lowerCamelCase							(   self						:Union[str, Any]	,  a						:str	,  a						:bool = False		)						->					str:
								print(a	,  flush=a	,  end="" if not stream_end else None		)







	def   _lowerCamelCase							(   self						:Optional[int]	,  a						:Optional[Any]		)						->					str:
								# This defines a "chinese character" as anything in the CJK Unicode block:
								#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
								#
								# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
								# despite its name. The modern Korean Hangul alphabet is a different block,
								# as is Japanese Hiragana and Katakana. Those alphabets are used to write
								# space-separated words, so they are not treated specially and handled
								# like the all of the other languages.
								if (
								    (cp >= 0x4e_00 and cp <= 0x9f_ff)
								    or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
								    or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
								    or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
								    or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
								    or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
								    or (cp >= 0xf9_00 and cp <= 0xfa_ff)
								    or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
								):  #
															return True

								return False




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	def __init__(   self						:Any	,  a						:"AutoTokenizer"	,  a						:bool = False	,  a						:Optional[float] = None	,  **a						:Dict		)						->					str:
								super().__init__(a	,  a	,  **a		)
								__UpperCamelCase      :  str       =   Queue()
								__UpperCamelCase      :  Dict       =   None
								__UpperCamelCase      :  Optional[int]       =   timeout







	def   _lowerCamelCase							(   self						:List[Any]	,  a						:str	,  a						:bool = False		)						->					Dict:
								self.text_queue.put(a	,  timeout=self.timeout		)
								if stream_end:
															self.text_queue.put(self.stop_signal	,  timeout=self.timeout		)







	def __iter__(   self						:Tuple		)						->					Tuple:
								return self







	def   _lowerCamelCase							(   self						:List[Any]		)						->					Dict:
								__UpperCamelCase      :  List[Any]       =   self.text_queue.get(timeout=self.timeout		)
								if value == self.stop_signal:
															raise StopIteration()
								else:
															return value