
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast




class 					lowerCamelCase__				(							unittest.TestCase):




	'''simple docstring'''




	def   _lowerCamelCase							(   self						:List[str]		)						->					Any:
								__UpperCamelCase      :  Optional[Any]       =   0







	@slow
	def   _lowerCamelCase							(   self						:Tuple		)						->					Dict:
								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
															__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(a		)
															self.assertIsNotNone(a		)
															self.assertIsInstance(a	,  (BertTokenizer, BertTokenizerFast)		)
															self.assertGreater(len(a		)	,  0		)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
															__UpperCamelCase      :  List[str]       =   AutoTokenizer.from_pretrained(a		)
															self.assertIsNotNone(a		)
															self.assertIsInstance(a	,  (GPTaTokenizer, GPTaTokenizerFast)		)
															self.assertGreater(len(a		)	,  0		)







	def   _lowerCamelCase							(   self						:int		)						->					Tuple:
								__UpperCamelCase      :  Optional[Any]       =   AutoTokenizer.from_pretrained(a		)
								self.assertIsInstance(a	,  (BertTokenizer, BertTokenizerFast)		)
								self.assertEqual(tokenizer.vocab_size	,  1_2		)







	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					Union[str, Any]:
								__UpperCamelCase      :  Optional[Any]       =   AutoTokenizer.from_pretrained(a		)
								self.assertIsInstance(a	,  (RobertaTokenizer, RobertaTokenizerFast)		)
								self.assertEqual(tokenizer.vocab_size	,  2_0		)







	def   _lowerCamelCase							(   self						:Tuple		)						->					int:
								__UpperCamelCase      :  List[str]       =   AutoConfig.from_pretrained(a		)
								self.assertIsInstance(a	,  a		)
								# Check that tokenizer_type ≠ model_type
								__UpperCamelCase      :  int       =   AutoTokenizer.from_pretrained(a	,  config=a		)
								self.assertIsInstance(a	,  (BertTokenizer, BertTokenizerFast)		)
								self.assertEqual(tokenizer.vocab_size	,  1_2		)







	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					Union[str, Any]:
								with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("./tests/fixtures/vocab.txt"	,  os.path.join(a	,  "vocab.txt"		)		)

															__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(a	,  tokenizer_type="bert"	,  use_fast=a		)
															self.assertIsInstance(a	,  a		)

								with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("./tests/fixtures/vocab.json"	,  os.path.join(a	,  "vocab.json"		)		)
															shutil.copy("./tests/fixtures/merges.txt"	,  os.path.join(a	,  "merges.txt"		)		)

															__UpperCamelCase      :  Any       =   AutoTokenizer.from_pretrained(a	,  tokenizer_type="gpt2"	,  use_fast=a		)
															self.assertIsInstance(a	,  a		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:Tuple		)						->					Any:
								with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("./tests/fixtures/vocab.txt"	,  os.path.join(a	,  "vocab.txt"		)		)

															__UpperCamelCase      :  List[Any]       =   AutoTokenizer.from_pretrained(a	,  tokenizer_type="bert"		)
															self.assertIsInstance(a	,  a		)

								with tempfile.TemporaryDirectory() as tmp_dir:
															shutil.copy("./tests/fixtures/vocab.json"	,  os.path.join(a	,  "vocab.json"		)		)
															shutil.copy("./tests/fixtures/merges.txt"	,  os.path.join(a	,  "merges.txt"		)		)

															__UpperCamelCase      :  str       =   AutoTokenizer.from_pretrained(a	,  tokenizer_type="gpt2"		)
															self.assertIsInstance(a	,  a		)







	def   _lowerCamelCase							(   self						:int		)						->					List[Any]:
								with pytest.raises(a		):
															AutoTokenizer.from_pretrained("./"	,  tokenizer_type="xxx"		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					str:
								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
															__UpperCamelCase      :  str       =   tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"		)
															self.assertIsInstance(a	,  (BertTokenizer, BertTokenizerFast)		)

															if isinstance(a	,  a		):
																						self.assertEqual(tokenizer.basic_tokenizer.do_lower_case	,  a		)
															else:
																						self.assertEqual(tokenizer.do_lower_case	,  a		)

															self.assertEqual(tokenizer.model_max_length	,  5_1_2		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:List[Any]		)						->					List[str]:
								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
															with self.assertRaisesRegex(
															    a	,  "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"	,  ):
																						__UpperCamelCase      :  Optional[Any]       =   tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"		)







	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					str:
								# tests: https://github.com/huggingface/transformers/pull/13251
								# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
								# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
								__UpperCamelCase      :  Optional[Any]       =   TOKENIZER_MAPPING.values()
								__UpperCamelCase      :  int       =   []

								for slow_tok, fast_tok in tokenizers:
															if slow_tok is not None:
																						tokenizer_names.append(slow_tok.__name__		)

															if fast_tok is not None:
																						tokenizer_names.append(fast_tok.__name__		)

								for tokenizer_name in tokenizer_names:
															# must find the right class
															tokenizer_class_from_name(a		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:List[str]		)						->					int:
								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"	,  use_fast=a		)	,  a		)
								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"		)	,  a		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:List[Any]		)						->					int:
								__UpperCamelCase      :  Union[str, Any]       =   AutoTokenizer.from_pretrained("distilbert-base-uncased"	,  do_lower_case=a		)
								__UpperCamelCase      :  Union[str, Any]       =   "Hello, world. How are you?"
								__UpperCamelCase      :  int       =   tokenizer.tokenize(a		)
								self.assertEqual("[UNK]"	,  tokens[0]		)

								__UpperCamelCase      :  List[str]       =   AutoTokenizer.from_pretrained("microsoft/mpnet-base"	,  do_lower_case=a		)
								__UpperCamelCase      :  Optional[int]       =   tokenizer.tokenize(a		)
								self.assertEqual("[UNK]"	,  tokens[0]		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:int		)						->					int:
								__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"		)
								self.assertEqual(type(a		)	,  a		)
								self.assertEqual(tokenizer.model_max_length	,  5_1_2		)
								self.assertEqual(tokenizer.vocab_size	,  3_0_0_0_0		)
								self.assertEqual(tokenizer.unk_token	,  "[UNK]"		)
								self.assertEqual(tokenizer.padding_side	,  "right"		)
								self.assertEqual(tokenizer.truncation_side	,  "right"		)







	def   _lowerCamelCase							(   self						:Any		)						->					int:
								__UpperCamelCase      :  Optional[Any]       =   AutoTokenizer.from_pretrained(a		)
								self.assertIsInstance(a	,  (BertTokenizer, BertTokenizerFast)		)
								with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(a		)
															__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(a		)

								self.assertIsInstance(a	,  tokenizer.__class__		)
								self.assertEqual(tokenizera.vocab_size	,  1_2		)







	def   _lowerCamelCase							(   self						:Dict		)						->					Optional[int]:
								__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained("ctrl"		)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(a	,  a		)







	def   _lowerCamelCase							(   self						:str		)						->					List[str]:
								# Check we can load the tokenizer config of an online model.
								__UpperCamelCase      :  Union[str, Any]       =   get_tokenizer_config("bert-base-cased"		)
								__UpperCamelCase      :  Optional[int]       =   config.pop("_commit_hash"	,  a		)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(a	,  {"do_lower_case": False}		)

								# This model does not have a tokenizer_config so we get back an empty dict.
								__UpperCamelCase      :  Optional[int]       =   get_tokenizer_config(a		)
								self.assertDictEqual(a	,  {}		)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained(a		)
								with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(a		)
															__UpperCamelCase      :  Optional[Any]       =   get_tokenizer_config(a		)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["tokenizer_class"]	,  "BertTokenizer"		)







	def   _lowerCamelCase							(   self						:Optional[int]		)						->					Optional[Any]:
								try:
															AutoConfig.register("custom"	,  a		)

															AutoTokenizer.register(a	,  slow_tokenizer_class=a		)
															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(a		):
																						AutoTokenizer.register(a	,  slow_tokenizer_class=a		)

															__UpperCamelCase      :  Optional[int]       =   CustomTokenizer.from_pretrained(a		)
															with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(a		)

																						__UpperCamelCase      :  int       =   AutoTokenizer.from_pretrained(a		)
																						self.assertIsInstance(a	,  a		)

								finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







	@require_tokenizers
	def   _lowerCamelCase							(   self						:Any		)						->					Union[str, Any]:
								try:
															AutoConfig.register("custom"	,  a		)

															# Can register in two steps
															AutoTokenizer.register(a	,  slow_tokenizer_class=a		)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,  (CustomTokenizer, None)		)
															AutoTokenizer.register(a	,  fast_tokenizer_class=a		)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,  (CustomTokenizer, CustomTokenizerFast)		)

															del TOKENIZER_MAPPING._extra_content[CustomConfig]
															# Can register in one step
															AutoTokenizer.register(
															    a	,  slow_tokenizer_class=a	,  fast_tokenizer_class=a		)
															self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,  (CustomTokenizer, CustomTokenizerFast)		)

															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(a		):
																						AutoTokenizer.register(a	,  fast_tokenizer_class=a		)

															# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
															# and that model does not have a tokenizer.json
															with tempfile.TemporaryDirectory() as tmp_dir:
																						__UpperCamelCase      :  Any       =   BertTokenizerFast.from_pretrained(a		)
																						bert_tokenizer.save_pretrained(a		)
																						__UpperCamelCase      :  Tuple       =   CustomTokenizerFast.from_pretrained(a		)

															with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(a		)

																						__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(a		)
																						self.assertIsInstance(a	,  a		)

																						__UpperCamelCase      :  str       =   AutoTokenizer.from_pretrained(a	,  use_fast=a		)
																						self.assertIsInstance(a	,  a		)

								finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







	def   _lowerCamelCase							(   self						:Tuple		)						->					Optional[int]:
								# If remote code is not set, we will time out when asking whether to load the model.
								with self.assertRaises(a		):
															__UpperCamelCase      :  List[Any]       =   AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(a		):
															__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a		)

								__UpperCamelCase      :  int       =   AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a		)
								self.assertTrue(tokenizer.special_attribute_present		)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer.save_pretrained(a		)
															__UpperCamelCase      :  List[Any]       =   AutoTokenizer.from_pretrained(a	,  trust_remote_code=a		)
								self.assertTrue(reloaded_tokenizer.special_attribute_present		)

								if is_tokenizers_available():
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizerFast"		)
															self.assertEqual(reloaded_tokenizer.__class__.__name__	,  "NewTokenizerFast"		)

															# Test we can also load the slow version
															__UpperCamelCase      :  List[str]       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a	,  use_fast=a		)
															self.assertTrue(tokenizer.special_attribute_present		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
															# Test tokenizer can be reloaded.
															with tempfile.TemporaryDirectory() as tmp_dir:
																						tokenizer.save_pretrained(a		)
																						__UpperCamelCase      :  List[str]       =   AutoTokenizer.from_pretrained(a	,  trust_remote_code=a	,  use_fast=a		)
															self.assertEqual(reloaded_tokenizer.__class__.__name__	,  "NewTokenizer"		)
															self.assertTrue(reloaded_tokenizer.special_attribute_present		)
								else:
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
															self.assertEqual(reloaded_tokenizer.__class__.__name__	,  "NewTokenizer"		)







	@require_tokenizers
	def   _lowerCamelCase							(   self						:Optional[int]		)						->					int:




								class 					lowerCamelCase__				(							__lowercase):




									'''simple docstring'''




									_A								=	False




								class 					lowerCamelCase__				(							__lowercase):




									'''simple docstring'''




									_A								=	NewTokenizer
									_A								=	False

								try:
															AutoConfig.register("custom"	,  a		)
															AutoTokenizer.register(a	,  slow_tokenizer_class=a		)
															AutoTokenizer.register(a	,  fast_tokenizer_class=a		)
															# If remote code is not set, the default is to use local
															__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizerFast"		)
															self.assertFalse(tokenizer.special_attribute_present		)
															__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"	,  use_fast=a		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
															self.assertFalse(tokenizer.special_attribute_present		)

															# If remote code is disabled, we load the local one.
															__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizerFast"		)
															self.assertFalse(tokenizer.special_attribute_present		)
															__UpperCamelCase      :  Dict       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a	,  use_fast=a		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
															self.assertFalse(tokenizer.special_attribute_present		)

															# If remote is enabled, we load from the Hub
															__UpperCamelCase      :  Any       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizerFast"		)
															self.assertTrue(tokenizer.special_attribute_present		)
															__UpperCamelCase      :  Union[str, Any]       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer"	,  trust_remote_code=a	,  use_fast=a		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
															self.assertTrue(tokenizer.special_attribute_present		)

								finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in TOKENIZER_MAPPING._extra_content:
																						del TOKENIZER_MAPPING._extra_content[CustomConfig]







	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					str:
								__UpperCamelCase      :  Optional[int]       =   AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"	,  trust_remote_code=a		)
								self.assertTrue(tokenizer.special_attribute_present		)
								if is_tokenizers_available():
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizerFast"		)

															# Test we can also load the slow version
															__UpperCamelCase      :  Optional[Any]       =   AutoTokenizer.from_pretrained(
															    "hf-internal-testing/test_dynamic_tokenizer_legacy"	,  trust_remote_code=a	,  use_fast=a		)
															self.assertTrue(tokenizer.special_attribute_present		)
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)
								else:
															self.assertEqual(tokenizer.__class__.__name__	,  "NewTokenizer"		)







	def   _lowerCamelCase							(   self						:Dict		)						->					Any:
								with self.assertRaisesRegex(
								    a	,  "bert-base is not a local folder and is not a valid model identifier"		):
															__UpperCamelCase      :  str       =   AutoTokenizer.from_pretrained("bert-base"		)







	def   _lowerCamelCase							(   self						:Dict		)						->					Optional[int]:
								with self.assertRaisesRegex(
								    a	,  r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"		):
															__UpperCamelCase      :  str       =   AutoTokenizer.from_pretrained(a	,  revision="aaaaaa"		)







	def   _lowerCamelCase							(   self						:List[str]		)						->					List[str]:
								# Make sure we have cached the tokenizer.
								__UpperCamelCase      :  int       =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
								with RequestCounter() as counter:
															__UpperCamelCase      :  str       =   AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
															self.assertEqual(counter.get_request_count	,  0		)
															self.assertEqual(counter.head_request_count	,  1		)
															self.assertEqual(counter.other_request_count	,  0		)

import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
		import tensorflow as tf

if is_tensorflow_text_available():
		from transformers.models.bert import TFBertTokenizer


lowercase    :   Union[str, Any]						=			['bert-base-uncased', 'bert-base-cased']
lowercase    :   Any						=			'hf-internal-testing/tiny-bert-tf-only'

if is_tf_available():




		class 					lowerCamelCase__				(							tf.keras.Model):




			'''simple docstring'''




			def __init__(   self						:List[Any]	,  a						:int		)						->					int:
										super().__init__()
										__UpperCamelCase      :  Any       =   tokenizer
										__UpperCamelCase      :  Union[str, Any]       =   AutoConfig.from_pretrained(a		)
										__UpperCamelCase      :  Dict       =   TFAutoModel.from_config(a		)







			def   _lowerCamelCase							(   self						:List[Any]	,  a						:Dict		)						->					Any:
										__UpperCamelCase      :  int       =   self.tokenizer(a		)
										__UpperCamelCase      :  int       =   self.bert(**a		)
										return out["pooler_output"]




@require_tf
@require_tensorflow_text
class 					lowerCamelCase__				(							unittest.TestCase):




	'''simple docstring'''




	def   _lowerCamelCase							(   self						:str		)						->					str:
								super().setUp()

								__UpperCamelCase      :  Dict       =   [
								    BertTokenizer.from_pretrained(a		) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
								]  # repeat for when fast_bert_tokenizer=false
								__UpperCamelCase      :  List[Any]       =   [TFBertTokenizer.from_pretrained(a		) for checkpoint in TOKENIZER_CHECKPOINTS] + [
								    TFBertTokenizer.from_pretrained(a	,  use_fast_bert_tokenizer=a		)
								    for checkpoint in TOKENIZER_CHECKPOINTS
								]
								assert len(self.tokenizers		) == len(self.tf_tokenizers		)

								__UpperCamelCase      :  Dict       =   [
								    "This is a straightforward English test sentence.",
								    "This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.",
								    "Now we're going to add some Chinese: 一 二 三 一二三",
								    "And some much more rare Chinese: 齉 堃 齉堃",
								    "Je vais aussi écrire en français pour tester les accents",
								    "Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ",
								]
								__UpperCamelCase      :  Dict       =   list(zip(self.test_sentences	,  self.test_sentences[::-1]		)		)







	def   _lowerCamelCase							(   self						:List[Any]		)						->					int:
								for tokenizer, tf_tokenizer in zip(self.tokenizers	,  self.tf_tokenizers		):
															for test_inputs in (self.test_sentences, self.paired_sentences):
																						__UpperCamelCase      :  str       =   tokenizer(a	,  return_tensors="tf"	,  padding="longest"		)
																						__UpperCamelCase      :  Any       =   tf_tokenizer(a		)

																						for key in python_outputs.keys():
																													self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape		)		)
																													self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]	,  tf.intaa		) == tf_outputs[key]		)		)







	@slow
	def   _lowerCamelCase							(   self						:Union[str, Any]		)						->					Any:
								for tf_tokenizer in self.tf_tokenizers:
															__UpperCamelCase      :  List[str]       =   tf_tokenizer(self.paired_sentences		)
															__UpperCamelCase      :  Union[str, Any]       =   tf_tokenizer(
															    text=[sentence[0] for sentence in self.paired_sentences]	,  text_pair=[sentence[1] for sentence in self.paired_sentences]	,  )
															for key in merged_outputs.keys():
																						self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]	,  tf.intaa		) == separated_outputs[key]		)		)







	@slow
	def   _lowerCamelCase							(   self						:Any		)						->					Optional[int]:
								for tf_tokenizer in self.tf_tokenizers:
															__UpperCamelCase      :  Optional[Any]       =   tf.function(a		)
															for test_inputs in (self.test_sentences, self.paired_sentences):
																						__UpperCamelCase      :  Tuple       =   tf.constant(a		)
																						__UpperCamelCase      :  List[str]       =   compiled_tokenizer(a		)
																						__UpperCamelCase      :  Any       =   tf_tokenizer(a		)

																						for key in eager_outputs.keys():
																													self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]		)		)







	@slow
	def   _lowerCamelCase							(   self						:List[Any]		)						->					Tuple:
								for tf_tokenizer in self.tf_tokenizers:
															__UpperCamelCase      :  str       =   ModelToSave(tokenizer=a		)
															__UpperCamelCase      :  List[Any]       =   tf.convert_to_tensor(self.test_sentences		)
															__UpperCamelCase      :  Optional[int]       =   model(a		)  # Build model with some sample inputs
															with TemporaryDirectory() as tempdir:
																						__UpperCamelCase      :  Optional[Any]       =   Path(a		) / "saved.model"
																						model.save(a		)
																						__UpperCamelCase      :  List[Any]       =   tf.keras.models.load_model(a		)
															__UpperCamelCase      :  Optional[Any]       =   loaded_model(a		)
															# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
															self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output		)		)	,  1E-5		)