





'''simple docstring'''






from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
						from ..models.auto import AutoTokenizer







class 		UpperCAmelCase   :


		"""simple docstring"""







		def   UpperCamelCase__      (				self    :					Optional[int]							,	UpperCamelCase__    :					Tuple  )     ->						List[Any]:
				raise NotImplementedError()




		def   UpperCamelCase__      (				self    :					Optional[int]  )     ->						Any:
				raise NotImplementedError()







class 		UpperCAmelCase   (				__A):


		"""simple docstring"""







		def __init__(				self    :					Union[str, Any]							,	UpperCamelCase__    :					Union[str, Any]							,	UpperCamelCase__    :					Any = False							,	**UpperCamelCase__    :					Optional[Any]  )     ->						Optional[Any]:
				_UpperCamelCase       			=tokenizer
				_UpperCamelCase       			=skip_prompt
				_UpperCamelCase       			=decode_kwargs

				# variables used in the streaming process
				_UpperCamelCase       			=[]
				_UpperCamelCase       			=0
				_UpperCamelCase       			=True


		def   UpperCamelCase__      (				self    :					str							,	UpperCamelCase__    :					int  )     ->						Tuple:





				if len(value.shape  ) > 1 and value.shape[0] > 1:
						raise ValueError('''TextStreamer only supports batch size 1'''  )
				elif len(value.shape  ) > 1:
						_UpperCamelCase       			=value[0]

				if self.skip_prompt and self.next_tokens_are_prompt:
						_UpperCamelCase       			=False
						return

				# Add the new token to the cache and decodes the entire thing.
				self.token_cache.extend(value.tolist()  )
				_UpperCamelCase       			=self.tokenizer.decode(self.token_cache							,	**self.decode_kwargs  )

				# After the symbol for a new line, we flush the cache.
				if text.endswith('''\n'''  ):
						_UpperCamelCase       			=text[self.print_len :]
						_UpperCamelCase       			=[]
						_UpperCamelCase       			=0
				# If the last token is a CJK character, we print the characters.
				elif len(UpperCamelCase__  ) > 0 and self._is_chinese_char(ord(text[-1]  )  ):
						_UpperCamelCase       			=text[self.print_len :]
						self.print_len += len(UpperCamelCase__  )
				# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
				# which may change with the subsequent token -- there are probably smarter ways to do this!)
				else:
						_UpperCamelCase       			=text[self.print_len : text.rfind(''' '''  ) + 1]
						self.print_len += len(UpperCamelCase__  )

				self.on_finalized_text(UpperCamelCase__  )


		def   UpperCamelCase__      (				self    :					Tuple  )     ->						Dict:





				if len(self.token_cache  ) > 0:
						_UpperCamelCase       			=self.tokenizer.decode(self.token_cache							,	**self.decode_kwargs  )
						_UpperCamelCase       			=text[self.print_len :]
						_UpperCamelCase       			=[]
						_UpperCamelCase       			=0
				else:
						_UpperCamelCase       			=''

				_UpperCamelCase       			=True
				self.on_finalized_text(UpperCamelCase__							,	stream_end=UpperCamelCase__  )


		def   UpperCamelCase__      (				self    :					str							,	UpperCamelCase__    :					Optional[int]							,	UpperCamelCase__    :					Dict = False  )     ->						Any:
				print(UpperCamelCase__							,	flush=UpperCamelCase__							,	end='''''' if not stream_end else None  )




		def   UpperCamelCase__      (				self    :					int							,	UpperCamelCase__    :					List[str]  )     ->						str:





				if (
				    (cp >= 0x4E00 and cp <= 0x9FFF)
				    or (cp >= 0x3400 and cp <= 0x4DBF)  #
				    or (cp >= 0x20000 and cp <= 0x2A6DF)  #
				    or (cp >= 0x2A700 and cp <= 0x2B73F)  #
				    or (cp >= 0x2B740 and cp <= 0x2B81F)  #
				    or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
				    or (cp >= 0xF900 and cp <= 0xFAFF)
				    or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
				):  #
						return True

				return False







class 		UpperCAmelCase   (				__A):


		"""simple docstring"""







		def __init__(				self    :					List[Any]							,	UpperCamelCase__    :					Optional[Any]							,	UpperCamelCase__    :					Optional[int] = False							,	UpperCamelCase__    :					List[Any] = None							,	**UpperCamelCase__    :					Optional[int]  )     ->						Optional[int]:
				super().__init__(UpperCamelCase__							,	UpperCamelCase__							,	**UpperCamelCase__  )
				_UpperCamelCase       			=Queue()
				_UpperCamelCase       			=None
				_UpperCamelCase       			=timeout


		def   UpperCamelCase__      (				self    :					Union[str, Any]							,	UpperCamelCase__    :					Optional[Any]							,	UpperCamelCase__    :					List[str] = False  )     ->						Dict:
				self.text_queue.put(UpperCamelCase__							,	timeout=self.timeout  )
				if stream_end:
						self.text_queue.put(self.stop_signal							,	timeout=self.timeout  )


		def __iter__(				self    :					Union[str, Any]  )     ->						Optional[int]:
				return self




		def   UpperCamelCase__      (				self    :					List[str]  )     ->						int:
				_UpperCamelCase       			=self.text_queue.get(timeout=self.timeout  )
				if value == self.stop_signal:
						raise StopIteration()
				else:
						return value













'''simple docstring'''






# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCamelCase						: Optional[int]		=   {
    'configuration_xmod': [
        'XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XmodConfig',
        'XmodOnnxConfig',
    ],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __lowerCamelCase						: int		=   [
          'XMOD_PRETRAINED_MODEL_ARCHIVE_LIST',
          'XmodForCausalLM',
          'XmodForMaskedLM',
          'XmodForMultipleChoice',
          'XmodForQuestionAnswering',
          'XmodForSequenceClassification',
          'XmodForTokenClassification',
          'XmodModel',
          'XmodPreTrainedModel',
      ]

if TYPE_CHECKING:
      from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_xmod import (
                XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
                XmodForCausalLM,
                XmodForMaskedLM,
                XmodForMultipleChoice,
                XmodForQuestionAnswering,
                XmodForSequenceClassification,
                XmodForTokenClassification,
                XmodModel,
                XmodPreTrainedModel,
            )

else:
      import sys

      __lowerCamelCase						: Dict		=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






