



"""simple docstring"""







import pytest

import datasets


# Import fixture modules as plugins
A    =			["""tests.fixtures.files""", """tests.fixtures.hub""", """tests.fixtures.fsspec"""]

def   lowerCAmelCase__			(     lowerCamelCase__				,      lowerCamelCase__			) ->							Any:







							for item in items:
														if any(marker in item.keywords for marker in ['integration', 'unit']			):
																					continue
														item.add_marker(pytest.mark.unit			)

def   lowerCAmelCase__			(     lowerCamelCase__			) ->							str:
							config.addinivalue_line('markers'				,      'torchaudio_latest: mark test to run with torchaudio>=0.12'			)

@pytest.fixture(autouse=lowerCamelCase__			)
def   lowerCAmelCase__			(     lowerCamelCase__				,      lowerCamelCase__			) ->							int:
							A         =					tmp_path_factory.getbasetemp() / 'cache'
							A         =					test_hf_cache_home / 'datasets'
							A         =					test_hf_cache_home / 'metrics'
							A         =					test_hf_cache_home / 'modules'
							monkeypatch.setattr('datasets.config.HF_DATASETS_CACHE'				,      str(lowerCamelCase__			)			)
							monkeypatch.setattr('datasets.config.HF_METRICS_CACHE'				,      str(lowerCamelCase__			)			)
							monkeypatch.setattr('datasets.config.HF_MODULES_CACHE'				,      str(lowerCamelCase__			)			)
							A         =					test_hf_datasets_cache / 'downloads'
							monkeypatch.setattr('datasets.config.DOWNLOADED_DATASETS_PATH'				,      str(lowerCamelCase__			)			)
							A         =					test_hf_datasets_cache / 'downloads' / 'extracted'
							monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH'				,      str(lowerCamelCase__			)			)

@pytest.fixture(autouse=lowerCamelCase__				,      scope='session'			)
def   lowerCAmelCase__			(     ) ->							Optional[int]:
							datasets.disable_progress_bar()

@pytest.fixture(autouse=lowerCamelCase__			)
def   lowerCAmelCase__			(     lowerCamelCase__			) ->							Any:
							monkeypatch.setattr('datasets.config.HF_UPDATE_DOWNLOAD_COUNTS'				,      lowerCamelCase__			)

@pytest.fixture
def   lowerCAmelCase__			(     lowerCamelCase__			) ->							List[Any]:
							monkeypatch.setattr('sqlalchemy.util.deprecations.SILENCE_UBER_WARNING'				,      lowerCamelCase__			)








"""simple docstring"""







import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


A    =			['bert-base-uncased', 'bert-base-cased']
A    =			'hf-internal-testing/tiny-bert-tf-only'

if is_tf_available():






							class      UpperCAmelCase__		(      tf.keras.Model     ):



									def __init__( self  :    Tuple      ,							snake_case  :    Optional[int]				)			->						List[str]:






																'''simple docstring'''

																super().__init__()
																A         =					tokenizer
																A         =					AutoConfig.from_pretrained(snake_case				)
																A         =					TFAutoModel.from_config(snake_case				)



									def 					A_		( self  :    Any      ,							snake_case  :    Optional[int]				)			->						Dict:






																'''simple docstring'''

																A         =					self.tokenizer(snake_case				)
																A         =					self.bert(**snake_case				)
																return out["pooler_output"]






@require_tf
@require_tensorflow_text
class      UpperCAmelCase__		(      unittest.TestCase     ):



		def 					A_		( self  :    int				)			->						Tuple:






									'''simple docstring'''

									super().setUp()

									A         =					[
									    BertTokenizer.from_pretrained(snake_case				) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
									]  # repeat for when fast_bert_tokenizer=false
									A         =					[TFBertTokenizer.from_pretrained(snake_case				) for checkpoint in TOKENIZER_CHECKPOINTS] + [
									    TFBertTokenizer.from_pretrained(snake_case      ,							use_fast_bert_tokenizer=snake_case				)
									    for checkpoint in TOKENIZER_CHECKPOINTS
									]
									assert len(self.tokenizers				) == len(self.tf_tokenizers				)

									A         =					[
									    'This is a straightforward English test sentence.',
									    'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
									    'Now we\'re going to add some Chinese: 一 二 三 一二三',
									    'And some much more rare Chinese: 齉 堃 齉堃',
									    'Je vais aussi écrire en français pour tester les accents',
									    'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
									]
									A         =					list(zip(self.test_sentences      ,							self.test_sentences[::-1]				)				)



		def 					A_		( self  :    List[Any]				)			->						Dict:






									'''simple docstring'''

									for tokenizer, tf_tokenizer in zip(self.tokenizers      ,							self.tf_tokenizers				):
																for test_inputs in (self.test_sentences, self.paired_sentences):
																							A         =					tokenizer(snake_case      ,							return_tensors='tf'      ,							padding='longest'				)
																							A         =					tf_tokenizer(snake_case				)

																							for key in python_outputs.keys():
																														self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape				)				)
																														self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]      ,							tf.intaa				) == tf_outputs[key]				)				)



		@slow
		def 					A_		( self  :    Tuple				)			->						Tuple:






									'''simple docstring'''

									for tf_tokenizer in self.tf_tokenizers:
																A         =					tf_tokenizer(self.paired_sentences				)
																A         =					tf_tokenizer(
																    text=[sentence[0] for sentence in self.paired_sentences]      ,							text_pair=[sentence[1] for sentence in self.paired_sentences]      ,							)
																for key in merged_outputs.keys():
																							self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]      ,							tf.intaa				) == separated_outputs[key]				)				)



		@slow
		def 					A_		( self  :    Any				)			->						Tuple:






									'''simple docstring'''

									for tf_tokenizer in self.tf_tokenizers:
																A         =					tf.function(snake_case				)
																for test_inputs in (self.test_sentences, self.paired_sentences):
																							A         =					tf.constant(snake_case				)
																							A         =					compiled_tokenizer(snake_case				)
																							A         =					tf_tokenizer(snake_case				)

																							for key in eager_outputs.keys():
																														self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]				)				)



		@slow
		def 					A_		( self  :    Optional[Any]				)			->						Optional[Any]:






									'''simple docstring'''

									for tf_tokenizer in self.tf_tokenizers:
																A         =					ModelToSave(tokenizer=snake_case				)
																A         =					tf.convert_to_tensor(self.test_sentences				)
																A         =					model(snake_case				)  # Build model with some sample inputs
																with TemporaryDirectory() as tempdir:
																							A         =					Path(snake_case				) / 'saved.model'
																							model.save(snake_case				)
																							A         =					tf.keras.models.load_model(snake_case				)
																A         =					loaded_model(snake_case				)
																# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
																self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output				)				)      ,							1E-5				)



