




import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


__UpperCamelCase   :			Optional[Any]          = threading.Lock()
__UpperCamelCase   :			Optional[logging.Handler]          = None

__UpperCamelCase   :			Dict          = {
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

__UpperCamelCase   :			Optional[int]          = logging.WARNING

__UpperCamelCase   :			Tuple          = True





def 						__A						(      ) ->			Any:
	a									=				os.getenv("""TRANSFORMERS_VERBOSITY"""				,     __lowerCamelCase       )
	if env_level_str:
		if env_level_str in log_levels:
			return log_levels[env_level_str]
		else:
			logging.getLogger().warning(
			    f'Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '
			    f'has to be one of: { ", ".join(log_levels.keys()       ) }'       )
	return _default_log_level





def 						__A						(      ) ->			str:
	return __name__.split("""."""       )[0]





def 						__A						(      ) ->			logging.Logger:
	return logging.getLogger(_get_library_name()       )





def 						__A						(      ) ->			None:
	global _default_handler

	with _lock:
		if _default_handler:
			# This library has already configured the library root logger.
			return
		a									=				logging.StreamHandler()  # Set sys.stderr as stream.
		a									=				sys.stderr.flush

		# Apply our default configuration to the library root logger.
		a									=				_get_library_root_logger()
		library_root_logger.addHandler(_default_handler       )
		library_root_logger.setLevel(_get_default_logging_level()       )
		a									=				False





def 						__A						(      ) ->			None:
	global _default_handler

	with _lock:
		if not _default_handler:
			return

		a									=				_get_library_root_logger()
		library_root_logger.removeHandler(_default_handler       )
		library_root_logger.setLevel(logging.NOTSET       )
		a									=				None





def 						__A						(      ) ->			str:
	return log_levels





def 						__A						(      __lowerCamelCase = None       ) ->			logging.Logger:

	if name is None:
		a									=				_get_library_name()

	_configure_library_root_logger()
	return logging.getLogger(__lowerCamelCase       )





def 						__A						(      ) ->			int:
	_configure_library_root_logger()
	return _get_library_root_logger().getEffectiveLevel()





def 						__A						(      __lowerCamelCase       ) ->			None:
	_configure_library_root_logger()
	_get_library_root_logger().setLevel(__lowerCamelCase       )





def 						__A						(      ) ->			Optional[int]:
	return set_verbosity(__lowerCamelCase       )





def 						__A						(      ) ->			Optional[Any]:
	return set_verbosity(__lowerCamelCase       )





def 						__A						(      ) ->			str:
	return set_verbosity(__lowerCamelCase       )





def 						__A						(      ) ->			List[str]:
	return set_verbosity(__lowerCamelCase       )





def 						__A						(      ) ->			None:
	_configure_library_root_logger()

	assert _default_handler is not None
	_get_library_root_logger().removeHandler(_default_handler       )





def 						__A						(      ) ->			None:
	_configure_library_root_logger()

	assert _default_handler is not None
	_get_library_root_logger().addHandler(_default_handler       )





def 						__A						(      __lowerCamelCase       ) ->			None:
	_configure_library_root_logger()

	assert handler is not None
	_get_library_root_logger().addHandler(__lowerCamelCase       )





def 						__A						(      __lowerCamelCase       ) ->			None:
	_configure_library_root_logger()

	assert handler is not None and handler not in _get_library_root_logger().handlers
	_get_library_root_logger().removeHandler(__lowerCamelCase       )





def 						__A						(      ) ->			None:
	_configure_library_root_logger()
	a									=				False





def 						__A						(      ) ->			None:
	_configure_library_root_logger()
	a									=				True





def 						__A						(      ) ->			None:
	a									=				_get_library_root_logger().handlers

	for handler in handlers:
		a									=				logging.Formatter("""[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"""       )
		handler.setFormatter(__lowerCamelCase       )





def 						__A						(      ) ->			None:
	a									=				_get_library_root_logger().handlers

	for handler in handlers:
		handler.setFormatter(__lowerCamelCase       )





def 						__A						(      self				,     *__lowerCamelCase				,     **__lowerCamelCase       ) ->			Union[str, Any]:
	a									=				os.getenv("""TRANSFORMERS_NO_ADVISORY_WARNINGS"""				,     __lowerCamelCase       )
	if no_advisory_warnings:
		return
	self.warning(*__lowerCamelCase				,     **__lowerCamelCase       )


__UpperCamelCase   :			List[str]          = warning_advice





@functools.lru_cache(__lowerCamelCase       )
def 						__A						(      self				,     *__lowerCamelCase				,     **__lowerCamelCase       ) ->			List[Any]:
	self.warning(*__lowerCamelCase				,     **__lowerCamelCase       )


__UpperCamelCase   :			Any          = warning_once

class 						__lowerCAmelCase      :
							def __init__(		self		:str						, *__magic_name__		:int						, **__magic_name__		:Optional[int]							):  # pylint: disable=unused-argument







								'''simple docstring'''







								a									=				args[0] if args else None
							def __iter__(		self		:Tuple							):







								'''simple docstring'''







								return iter(self._iterator							)
							def __getattr__(		self		:Dict						, __magic_name__		:Optional[Any]							):







								'''simple docstring'''







								def empty_fn(*__magic_name__		:Optional[int]						, **__magic_name__		:Dict							):  # pylint: disable=unused-argument
									return

								return empty_fn
							def __enter__(		self		:Union[str, Any]							):







								'''simple docstring'''







								return self





							def __exit__(		self		:str						, __magic_name__		:Tuple						, __magic_name__		:str						, __magic_name__		:Dict							):







								'''simple docstring'''







								return







class 						__lowerCAmelCase      :
							def __call__(		self		:Optional[Any]						, *__magic_name__		:Tuple						, **__magic_name__		:List[str]							):







								'''simple docstring'''







								if _tqdm_active:
									return tqdm_lib.tqdm(*__magic_name__						, **__magic_name__							)
								else:
									return EmptyTqdm(*__magic_name__						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, *__magic_name__		:List[str]						, **__magic_name__		:Tuple							):







								'''simple docstring'''







								a									=				None
								if _tqdm_active:
									return tqdm_lib.tqdm.set_lock(*__magic_name__						, **__magic_name__							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								if _tqdm_active:
									return tqdm_lib.tqdm.get_lock()







__UpperCamelCase   :			Dict          = _tqdm_cls()





def 						__A						(      ) ->			bool:
	global _tqdm_active
	return bool(_tqdm_active       )





def 						__A						(      ) ->			Tuple:
	global _tqdm_active
	a									=				True
	hf_hub_utils.enable_progress_bars()





def 						__A						(      ) ->			Tuple:
	global _tqdm_active
	a									=				False
	hf_hub_utils.disable_progress_bars()







import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 						__lowerCAmelCase      (      __magic_name__ ,			unittest.TestCase					):
							UpperCamelCase__   	=    TransfoXLTokenizer
							UpperCamelCase__   	=    False
							UpperCamelCase__   	=    False
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								super().setUp()

								a									=				[
								    """<unk>""",
								    """[CLS]""",
								    """[SEP]""",
								    """want""",
								    """unwanted""",
								    """wa""",
								    """un""",
								    """running""",
								    """,""",
								    """low""",
								    """l""",
								]
								a									=				os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES["""vocab_file"""]							)
								with open(self.vocab_file						, """w"""						, encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)
							def  lowerCamelCase__				(		self		:List[Any]						, **__magic_name__		:Optional[int]							):







								'''simple docstring'''







								a									=				True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname						, **__magic_name__							)
							def  lowerCamelCase__				(		self		:Union[str, Any]						, __magic_name__		:int							):







								'''simple docstring'''







								a									=				"""<unk> UNwanted , running"""
								a									=				"""<unk> unwanted, running"""
								return input_text, output_text
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(vocab_file=self.vocab_file						, lower_case=__magic_name__							)

								a									=				tokenizer.tokenize("""<unk> UNwanted , running"""							)
								self.assertListEqual(__magic_name__						, ["""<unk>""", """unwanted""", """,""", """running"""]							)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__							)						, [0, 4, 8, 7]							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""hello""", """!""", """how""", """are""", """you""", """?"""]							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """							)						, ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]							)
							def  lowerCamelCase__				(		self		:Optional[int]							):







								'''simple docstring'''







								a									=				TransfoXLTokenizer(lower_case=__magic_name__							)
								a									=				"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
								a									=				[
								    """Hello""",
								    """(""",
								    """bracket""",
								    """)""",
								    """and""",
								    """side""",
								    """@-@""",
								    """scrolled""",
								    """[""",
								    """and""",
								    """]""",
								    """Henry""",
								    """'s""",
								    """$""",
								    """5""",
								    """@,@""",
								    """000""",
								    """with""",
								    """3""",
								    """@.@""",
								    """34""",
								    """m""",
								    """.""",
								    """What""",
								    """'s""",
								    """up""",
								    """!""",
								    """?""",
								]

								self.assertListEqual(tokenizer.tokenize(__magic_name__							)						, __magic_name__							)

								self.assertEqual(tokenizer.convert_tokens_to_string(__magic_name__							)						, __magic_name__							)





							def  lowerCamelCase__				(		self		:List[Any]							):







								'''simple docstring'''







								a									=				self.get_tokenizer()
								a									=				len(__magic_name__							)

								tokenizer.add_tokens(["""new1""", """new2"""]							)
								tokenizer.move_added_token("""new1"""						, 1							)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(__magic_name__							)						, original_len + 2							)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode("""new1"""							)						, [1]							)
								self.assertEqual(tokenizer.decode([1]							)						, """new1"""							)

