





"""simple docstring"""






import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


A 			=  threading.Lock()
A 			=  None

A 			=  {
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

A 			=  logging.WARNING

A 			=  True

def     __A      (			)		->		Union[str, Any]:
				__a :						List[str]			       =  os.getenv('''TRANSFORMERS_VERBOSITY'''   ,       a_)
				if env_level_str:
								if env_level_str in log_levels:
												return log_levels[env_level_str]
								else:
												logging.getLogger().warning(
												    F"""Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, """
												    F"""has to be one of: { ", ".join(log_levels.keys()) }""")
				return _default_log_level

def     __A      (			)		->		str:
				return __name__.split('''.''')[0]

def     __A      (			)		->		logging.Logger:
				return logging.getLogger(_get_library_name())

def     __A      (			)		->		None:
				global _default_handler

				with _lock:
								if _default_handler:
												# This library has already configured the library root logger.
												return
								__a :						int			       =  logging.StreamHandler()  # Set sys.stderr as stream.
								__a :						Optional[Any]			       =  sys.stderr.flush

								# Apply our default configuration to the library root logger.
								__a :						int			       =  _get_library_root_logger()
								library_root_logger.addHandler(_default_handler)
								library_root_logger.setLevel(_get_default_logging_level())
								__a :						str			       =  False

def     __A      (			)		->		None:
				global _default_handler

				with _lock:
								if not _default_handler:
												return

								__a :						List[str]			       =  _get_library_root_logger()
								library_root_logger.removeHandler(_default_handler)
								library_root_logger.setLevel(logging.NOTSET)
								__a :						int			       =  None

def     __A      (			)		->		Optional[Any]:
				return log_levels

def     __A      (			a_						:Optional[str] = None)		->		logging.Logger:

				if name is None:
								__a :						Dict			       =  _get_library_name()

				_configure_library_root_logger()
				return logging.getLogger(a_)

def     __A      (			)		->		int:
				_configure_library_root_logger()
				return _get_library_root_logger().getEffectiveLevel()

def     __A      (			a_						:int)		->		None:
				_configure_library_root_logger()
				_get_library_root_logger().setLevel(a_)

def     __A      (			)		->		int:
				return set_verbosity(a_)

def     __A      (			)		->		List[Any]:
				return set_verbosity(a_)

def     __A      (			)		->		Optional[int]:
				return set_verbosity(a_)

def     __A      (			)		->		int:
				return set_verbosity(a_)

def     __A      (			)		->		None:
				_configure_library_root_logger()

				assert _default_handler is not None
				_get_library_root_logger().removeHandler(_default_handler)

def     __A      (			)		->		None:
				_configure_library_root_logger()

				assert _default_handler is not None
				_get_library_root_logger().addHandler(_default_handler)

def     __A      (			a_						:logging.Handler)		->		None:
				_configure_library_root_logger()

				assert handler is not None
				_get_library_root_logger().addHandler(a_)

def     __A      (			a_						:logging.Handler)		->		None:
				_configure_library_root_logger()

				assert handler is not None and handler not in _get_library_root_logger().handlers
				_get_library_root_logger().removeHandler(a_)

def     __A      (			)		->		None:
				_configure_library_root_logger()
				__a :						Dict			       =  False

def     __A      (			)		->		None:
				_configure_library_root_logger()
				__a :						Optional[Any]			       =  True

def     __A      (			)		->		None:
				__a :						Any			       =  _get_library_root_logger().handlers

				for handler in handlers:
								__a :						str			       =  logging.Formatter('''[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s''')
								handler.setFormatter(a_)

def     __A      (			)		->		None:
				__a :						List[str]			       =  _get_library_root_logger().handlers

				for handler in handlers:
								handler.setFormatter(a_)

def     __A      (			self						:Optional[Any]   ,       *a_						:str   ,       **a_						:Optional[int])		->		int:
				__a :						Any			       =  os.getenv('''TRANSFORMERS_NO_ADVISORY_WARNINGS'''   ,       a_)
				if no_advisory_warnings:
								return
				self.warning(*a_   ,       **a_)


A 			=  warning_advice

@functools.lru_cache(a_)
def     __A      (			self						:Optional[Any]   ,       *a_						:int   ,       **a_						:List[Any])		->		List[str]:
				self.warning(*a_   ,       **a_)


A 			=  warning_once




class 				__lowercase			:







					'''simple docstring'''




					def __init__(						self	,			*_UpperCAmelCase	,			**_UpperCAmelCase     ):  # pylint: disable=unused-argument
									__a :						List[Any]			       =  args[0] if args else None







					def __iter__(						self     ):
									return iter(self._iterator     )







					def __getattr__(						self	,			_UpperCAmelCase     ):

									def empty_fn(*_UpperCAmelCase	,			**_UpperCAmelCase     ):  # pylint: disable=unused-argument
													return

									return empty_fn







					def __enter__(						self     ):
									return self







					def __exit__(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									return






class 				__lowercase			:







					'''simple docstring'''




					def __call__(						self	,			*_UpperCAmelCase	,			**_UpperCAmelCase     ):
									if _tqdm_active:
													return tqdm_lib.tqdm(*_UpperCAmelCase	,			**_UpperCAmelCase     )
									else:
													return EmptyTqdm(*_UpperCAmelCase	,			**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			*_UpperCAmelCase	,			**_UpperCAmelCase     ):
									__a :						Any			       =  None
									if _tqdm_active:
													return tqdm_lib.tqdm.set_lock(*_UpperCAmelCase	,			**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									if _tqdm_active:
													return tqdm_lib.tqdm.get_lock()






A 			=  _tqdm_cls()

def     __A      (			)		->		bool:
				global _tqdm_active
				return bool(_tqdm_active)

def     __A      (			)		->		str:
				global _tqdm_active
				__a :						str			       =  True
				hf_hub_utils.enable_progress_bars()






def     __A      (			)		->		int:
				global _tqdm_active
				__a :						int			       =  False
				hf_hub_utils.disable_progress_bars()






"""simple docstring"""






import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
		import tensorflow as tf

if is_tensorflow_text_available():
		from transformers.models.bert import TFBertTokenizer


A 			=  ['''bert-base-uncased''', '''bert-base-cased''']
A 			=  '''hf-internal-testing/tiny-bert-tf-only'''

if is_tf_available():




		class 				__lowercase			(			tf.keras.Model							):







							'''simple docstring'''




							def __init__(						self	,			_UpperCAmelCase     ):
											super().__init__()
											__a :						Any			       =  tokenizer
											__a :						Optional[Any]			       =  AutoConfig.from_pretrained(_UpperCAmelCase     )
											__a :						str			       =  TFAutoModel.from_config(_UpperCAmelCase     )







							def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
											__a :						Any			       =  self.tokenizer(_UpperCAmelCase     )
											__a :						int			       =  self.bert(**_UpperCAmelCase     )
											return out["pooler_output"]




@require_tf
@require_tensorflow_text
class 				__lowercase			(			unittest.TestCase							):







					'''simple docstring'''




					def  _lowerCamelCase		(						self     ):
									super().setUp()

									__a :						Any			       =  [
									    BertTokenizer.from_pretrained(_UpperCAmelCase     ) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
									]  # repeat for when fast_bert_tokenizer=false
									__a :						Union[str, Any]			       =  [TFBertTokenizer.from_pretrained(_UpperCAmelCase     ) for checkpoint in TOKENIZER_CHECKPOINTS] + [
									    TFBertTokenizer.from_pretrained(_UpperCAmelCase	,			use_fast_bert_tokenizer=_UpperCAmelCase     )
									    for checkpoint in TOKENIZER_CHECKPOINTS
									]
									assert len(self.tokenizers     ) == len(self.tf_tokenizers     )

									__a :						Tuple			       =  [
									    '''This is a straightforward English test sentence.''',
									    '''This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.''',
									    '''Now we\'re going to add some Chinese: 一 二 三 一二三''',
									    '''And some much more rare Chinese: 齉 堃 齉堃''',
									    '''Je vais aussi écrire en français pour tester les accents''',
									    '''Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ''',
									]
									__a :						Any			       =  list(zip(self.test_sentences	,			self.test_sentences[::-1]     )     )







					def  _lowerCamelCase		(						self     ):
									for tokenizer, tf_tokenizer in zip(self.tokenizers	,			self.tf_tokenizers     ):
													for test_inputs in (self.test_sentences, self.paired_sentences):
																	__a :						List[Any]			       =  tokenizer(_UpperCAmelCase	,			return_tensors='''tf'''	,			padding='''longest'''     )
																	__a :						List[str]			       =  tf_tokenizer(_UpperCAmelCase     )

																	for key in python_outputs.keys():
																					self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape     )     )
																					self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]	,			tf.intaa     ) == tf_outputs[key]     )     )







					@slow
					def  _lowerCamelCase		(						self     ):
									for tf_tokenizer in self.tf_tokenizers:
													__a :						Dict			       =  tf_tokenizer(self.paired_sentences     )
													__a :						str			       =  tf_tokenizer(
													    text=[sentence[0] for sentence in self.paired_sentences]	,			text_pair=[sentence[1] for sentence in self.paired_sentences]	,			)
													for key in merged_outputs.keys():
																	self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]	,			tf.intaa     ) == separated_outputs[key]     )     )







					@slow
					def  _lowerCamelCase		(						self     ):
									for tf_tokenizer in self.tf_tokenizers:
													__a :						Tuple			       =  tf.function(_UpperCAmelCase     )
													for test_inputs in (self.test_sentences, self.paired_sentences):
																	__a :						List[str]			       =  tf.constant(_UpperCAmelCase     )
																	__a :						Tuple			       =  compiled_tokenizer(_UpperCAmelCase     )
																	__a :						Union[str, Any]			       =  tf_tokenizer(_UpperCAmelCase     )

																	for key in eager_outputs.keys():
																					self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]     )     )







					@slow
					def  _lowerCamelCase		(						self     ):
									for tf_tokenizer in self.tf_tokenizers:
													__a :						Dict			       =  ModelToSave(tokenizer=_UpperCAmelCase     )
													__a :						Tuple			       =  tf.convert_to_tensor(self.test_sentences     )
													__a :						List[str]			       =  model(_UpperCAmelCase     )  # Build model with some sample inputs
													with TemporaryDirectory() as tempdir:
																	__a :						Tuple			       =  Path(_UpperCAmelCase     ) / '''saved.model'''
																	model.save(_UpperCAmelCase     )
																	__a :						Tuple			       =  tf.keras.models.load_model(_UpperCAmelCase     )
													__a :						int			       =  loaded_model(_UpperCAmelCase     )
													# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
													self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output     )     )	,			1e-5     )