

"""simple docstring"""






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast







class       snake_case__  ( unittest.TestCase     ):







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				0







			@slow
			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
													self.assertIsNotNone(UpperCamelCase__  )
													self.assertIsInstance(UpperCamelCase__			,  (BertTokenizer, BertTokenizerFast)  )
													self.assertGreater(len(UpperCamelCase__  )			,  0  )

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
													UpperCAmelCase			:   List[str]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
													self.assertIsNotNone(UpperCamelCase__  )
													self.assertIsInstance(UpperCamelCase__			,  (GPTaTokenizer, GPTaTokenizerFast)  )
													self.assertGreater(len(UpperCamelCase__  )			,  0  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
								self.assertIsInstance(UpperCamelCase__			,  (BertTokenizer, BertTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
								self.assertIsInstance(UpperCamelCase__			,  (RobertaTokenizer, RobertaTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  20  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				AutoConfig.from_pretrained(UpperCamelCase__  )
								self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )
								# Check that tokenizer_type ≠ model_type
								UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  config=UpperCamelCase__  )
								self.assertIsInstance(UpperCamelCase__			,  (BertTokenizer, BertTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(UpperCamelCase__			,  "vocab.txt"  )  )

													UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  tokenizer_type="bert"			,  use_fast=UpperCamelCase__  )
													self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )

								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(UpperCamelCase__			,  "vocab.json"  )  )
													shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(UpperCamelCase__			,  "merges.txt"  )  )

													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  tokenizer_type="gpt2"			,  use_fast=UpperCamelCase__  )
													self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(UpperCamelCase__			,  "vocab.txt"  )  )

													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  tokenizer_type="bert"  )
													self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )

								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(UpperCamelCase__			,  "vocab.json"  )  )
													shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(UpperCamelCase__			,  "merges.txt"  )  )

													UpperCAmelCase			:   Dict      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  tokenizer_type="gpt2"  )
													self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								with pytest.raises(UpperCamelCase__  ):
													AutoTokenizer.from_pretrained("./"			,  tokenizer_type="xxx"  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
													UpperCAmelCase			:   Any      =				tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"  )
													self.assertIsInstance(UpperCamelCase__			,  (BertTokenizer, BertTokenizerFast)  )

													if isinstance(UpperCamelCase__			,  UpperCamelCase__  ):
																		self.assertEqual(tokenizer.basic_tokenizer.do_lower_case			,  UpperCamelCase__  )
													else:
																		self.assertEqual(tokenizer.do_lower_case			,  UpperCamelCase__  )

													self.assertEqual(tokenizer.model_max_length			,  5_12  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
													with self.assertRaisesRegex(
													    UpperCamelCase__			,  "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"			,  ):
																		UpperCAmelCase			:   List[Any]      =				tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				TOKENIZER_MAPPING.values()
								UpperCAmelCase			:   int      =				[]

								for slow_tok, fast_tok in tokenizers:
													if slow_tok is not None:
																		tokenizer_names.append(slow_tok.__name__  )

													if fast_tok is not None:
																		tokenizer_names.append(fast_tok.__name__  )

								for tokenizer_name in tokenizer_names:
													# must find the right class
													tokenizer_class_from_name(UpperCamelCase__  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"			,  use_fast=UpperCamelCase__  )			,  UpperCamelCase__  )
								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"  )			,  UpperCamelCase__  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained("distilbert-base-uncased"			,  do_lower_case=UpperCamelCase__  )
								UpperCAmelCase			:   int      =				"Hello, world. How are you?"
								UpperCAmelCase			:   List[str]      =				tokenizer.tokenize(UpperCamelCase__  )
								self.assertEqual("[UNK]"			,  tokens[0]  )

								UpperCAmelCase			:   Any      =				AutoTokenizer.from_pretrained("microsoft/mpnet-base"			,  do_lower_case=UpperCamelCase__  )
								UpperCAmelCase			:   Union[str, Any]      =				tokenizer.tokenize(UpperCamelCase__  )
								self.assertEqual("[UNK]"			,  tokens[0]  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"  )
								self.assertEqual(type(UpperCamelCase__  )			,  UpperCamelCase__  )
								self.assertEqual(tokenizer.model_max_length			,  5_12  )
								self.assertEqual(tokenizer.vocab_size			,  3_00_00  )
								self.assertEqual(tokenizer.unk_token			,  "[UNK]"  )
								self.assertEqual(tokenizer.padding_side			,  "right"  )
								self.assertEqual(tokenizer.truncation_side			,  "right"  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
								self.assertIsInstance(UpperCamelCase__			,  (BertTokenizer, BertTokenizerFast)  )
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(UpperCamelCase__  )
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )

								self.assertIsInstance(UpperCamelCase__			,  tokenizer.__class__  )
								self.assertEqual(tokenizera.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained("ctrl"  )
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				get_tokenizer_config("bert-base-cased"  )
								UpperCAmelCase			:   Any      =				config.pop("_commit_hash"			,  UpperCamelCase__  )
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(UpperCamelCase__			,  {"do_lower_case": False}  )

								# This model does not have a tokenizer_config so we get back an empty dict.
								UpperCAmelCase			:   Union[str, Any]      =				get_tokenizer_config(UpperCamelCase__  )
								self.assertDictEqual(UpperCamelCase__			,  {}  )

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								UpperCAmelCase			:   Optional[int]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(UpperCamelCase__  )
													UpperCAmelCase			:   Union[str, Any]      =				get_tokenizer_config(UpperCamelCase__  )

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["tokenizer_class"]			,  "BertTokenizer"  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								try:
													AutoConfig.register("custom"			,  UpperCamelCase__  )

													AutoTokenizer.register(UpperCamelCase__			,  slow_tokenizer_class=UpperCamelCase__  )
													# Trying to register something existing in the Transformers library will raise an error
													with self.assertRaises(UpperCamelCase__  ):
																		AutoTokenizer.register(UpperCamelCase__			,  slow_tokenizer_class=UpperCamelCase__  )

													UpperCAmelCase			:   Tuple      =				CustomTokenizer.from_pretrained(UpperCamelCase__  )
													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(UpperCamelCase__  )

																		UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
																		self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								try:
													AutoConfig.register("custom"			,  UpperCamelCase__  )

													# Can register in two steps
													AutoTokenizer.register(UpperCamelCase__			,  slow_tokenizer_class=UpperCamelCase__  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, None)  )
													AutoTokenizer.register(UpperCamelCase__			,  fast_tokenizer_class=UpperCamelCase__  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)  )

													del TOKENIZER_MAPPING._extra_content[CustomConfig]
													# Can register in one step
													AutoTokenizer.register(
													    UpperCamelCase__			,  slow_tokenizer_class=UpperCamelCase__			,  fast_tokenizer_class=UpperCamelCase__  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)  )

													# Trying to register something existing in the Transformers library will raise an error
													with self.assertRaises(UpperCamelCase__  ):
																		AutoTokenizer.register(UpperCamelCase__			,  fast_tokenizer_class=UpperCamelCase__  )

													# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
													# and that model does not have a tokenizer.json
													with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase			:   Optional[int]      =				BertTokenizerFast.from_pretrained(UpperCamelCase__  )
																		bert_tokenizer.save_pretrained(UpperCamelCase__  )
																		UpperCAmelCase			:   List[str]      =				CustomTokenizerFast.from_pretrained(UpperCamelCase__  )

													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(UpperCamelCase__  )

																		UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__  )
																		self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )

																		UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  use_fast=UpperCamelCase__  )
																		self.assertIsInstance(UpperCamelCase__			,  UpperCamelCase__  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								with self.assertRaises(UpperCamelCase__  ):
													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"  )
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(UpperCamelCase__  ):
													UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__  )

								UpperCAmelCase			:   Optional[int]      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__  )
								self.assertTrue(tokenizer.special_attribute_present  )
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(UpperCamelCase__  )
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  trust_remote_code=UpperCamelCase__  )
								self.assertTrue(reloaded_tokenizer.special_attribute_present  )

								if is_tokenizers_available():
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizerFast"  )

													# Test we can also load the slow version
													UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__			,  use_fast=UpperCamelCase__  )
													self.assertTrue(tokenizer.special_attribute_present  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													# Test tokenizer can be reloaded.
													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(UpperCamelCase__  )
																		UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  trust_remote_code=UpperCamelCase__			,  use_fast=UpperCamelCase__  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertTrue(reloaded_tokenizer.special_attribute_present  )
								else:
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								class       snake_case__  ( _A     ):
											SCREAMING_SNAKE_CASE__     				=						False







								class       snake_case__  ( _A     ):
											SCREAMING_SNAKE_CASE__     				=						NewTokenizer
											SCREAMING_SNAKE_CASE__     				=						False

								try:
													AutoConfig.register("custom"			,  UpperCamelCase__  )
													AutoTokenizer.register(UpperCamelCase__			,  slow_tokenizer_class=UpperCamelCase__  )
													AutoTokenizer.register(UpperCamelCase__			,  fast_tokenizer_class=UpperCamelCase__  )
													# If remote code is not set, the default is to use local
													UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertFalse(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   Optional[int]      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  use_fast=UpperCamelCase__  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertFalse(tokenizer.special_attribute_present  )

													# If remote code is disabled, we load the local one.
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertFalse(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__			,  use_fast=UpperCamelCase__  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertFalse(tokenizer.special_attribute_present  )

													# If remote is enabled, we load from the Hub
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertTrue(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=UpperCamelCase__			,  use_fast=UpperCamelCase__  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertTrue(tokenizer.special_attribute_present  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=UpperCamelCase__  )
								self.assertTrue(tokenizer.special_attribute_present  )
								if is_tokenizers_available():
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )

													# Test we can also load the slow version
													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=UpperCamelCase__			,  use_fast=UpperCamelCase__  )
													self.assertTrue(tokenizer.special_attribute_present  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
								else:
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								with self.assertRaisesRegex(
								    UpperCamelCase__			,  "bert-base is not a local folder and is not a valid model identifier"  ):
													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained("bert-base"  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								with self.assertRaisesRegex(
								    UpperCamelCase__			,  r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"  ):
													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(UpperCamelCase__			,  revision="aaaaaa"  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
								with RequestCounter() as counter:
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
													self.assertEqual(counter.get_request_count			,  0  )
													self.assertEqual(counter.head_request_count			,  1  )
													self.assertEqual(counter.other_request_count			,  0  )





"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel







@require_tf
class       snake_case__  :
			SCREAMING_SNAKE_CASE__     				=						BlenderbotSmallConfig
			SCREAMING_SNAKE_CASE__     				=						{}
			SCREAMING_SNAKE_CASE__     				=						'''gelu'''







			def __init__(			self			: Tuple			,  lowercase			: Any			,  lowercase			: Optional[Any]=13			,  lowercase			: Any=7			,  lowercase			: List[Any]=True			,  lowercase			: List[str]=False			,  lowercase			: Optional[Any]=99			,  lowercase			: Union[str, Any]=32			,  lowercase			: List[Any]=2			,  lowercase			: Tuple=4			,  lowercase			: Union[str, Any]=37			,  lowercase			: str=0.1			,  lowercase			: Optional[Any]=0.1			,  lowercase			: Optional[Any]=20			,  lowercase			: str=2			,  lowercase			: int=1			,  lowercase			: Optional[int]=0			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				parent
								UpperCAmelCase			:   Union[str, Any]      =				batch_size
								UpperCAmelCase			:   Tuple      =				seq_length
								UpperCAmelCase			:   Dict      =				is_training
								UpperCAmelCase			:   str      =				use_labels
								UpperCAmelCase			:   Union[str, Any]      =				vocab_size
								UpperCAmelCase			:   Tuple      =				hidden_size
								UpperCAmelCase			:   List[Any]      =				num_hidden_layers
								UpperCAmelCase			:   Union[str, Any]      =				num_attention_heads
								UpperCAmelCase			:   Dict      =				intermediate_size

								UpperCAmelCase			:   Optional[int]      =				hidden_dropout_prob
								UpperCAmelCase			:   Optional[Any]      =				attention_probs_dropout_prob
								UpperCAmelCase			:   Dict      =				max_position_embeddings
								UpperCAmelCase			:   List[Any]      =				eos_token_id
								UpperCAmelCase			:   List[Any]      =				pad_token_id
								UpperCAmelCase			:   Union[str, Any]      =				bos_token_id







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				ids_tensor([self.batch_size, self.seq_length - 1]			,  self.vocab_size  )
								UpperCAmelCase			:   Optional[int]      =				tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size  )			,  1  )
								UpperCAmelCase			:   Optional[int]      =				tf.concat([input_ids, eos_tensor]			,  axis=1  )

								UpperCAmelCase			:   str      =				ids_tensor([self.batch_size, self.seq_length]			,  self.vocab_size  )

								UpperCAmelCase			:   int      =				self.config_cls(
								    vocab_size=self.vocab_size			,  d_model=self.hidden_size			,  encoder_layers=self.num_hidden_layers			,  decoder_layers=self.num_hidden_layers			,  encoder_attention_heads=self.num_attention_heads			,  decoder_attention_heads=self.num_attention_heads			,  encoder_ffn_dim=self.intermediate_size			,  decoder_ffn_dim=self.intermediate_size			,  dropout=self.hidden_dropout_prob			,  attention_dropout=self.attention_probs_dropout_prob			,  max_position_embeddings=self.max_position_embeddings			,  eos_token_ids=[2]			,  bos_token_id=self.bos_token_id			,  pad_token_id=self.pad_token_id			,  decoder_start_token_id=self.pad_token_id			,  **self.config_updates			,  )
								UpperCAmelCase			:   Optional[Any]      =				prepare_blenderbot_small_inputs_dict(lowercase			,  lowercase			,  lowercase  )
								return config, inputs_dict







			def 			__lowerCAmelCase				(			self			: int			,  lowercase			: List[Any]			,  lowercase			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				TFBlenderbotSmallModel(config=lowercase  ).get_decoder()
								UpperCAmelCase			:   List[Any]      =				inputs_dict["input_ids"]

								UpperCAmelCase			:   int      =				input_ids[:1, :]
								UpperCAmelCase			:   Union[str, Any]      =				inputs_dict["attention_mask"][:1, :]
								UpperCAmelCase			:   Union[str, Any]      =				inputs_dict["head_mask"]
								UpperCAmelCase			:   List[Any]      =				1

								# first forward pass
								UpperCAmelCase			:   Optional[Any]      =				model(lowercase			,  attention_mask=lowercase			,  head_mask=lowercase			,  use_cache=lowercase  )

								UpperCAmelCase      ,							UpperCAmelCase			:   List[str]      =				outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								UpperCAmelCase			:   Any      =				ids_tensor((self.batch_size, 3)			,  config.vocab_size  )
								UpperCAmelCase			:   Optional[int]      =				tf.cast(ids_tensor((self.batch_size, 3)			,  2  )			,  tf.inta  )

								# append to next input_ids and
								UpperCAmelCase			:   Union[str, Any]      =				tf.concat([input_ids, next_tokens]			,  axis=-1  )
								UpperCAmelCase			:   str      =				tf.concat([attention_mask, next_attn_mask]			,  axis=-1  )

								UpperCAmelCase			:   Optional[Any]      =				model(lowercase			,  attention_mask=lowercase  )[0]
								UpperCAmelCase			:   List[str]      =				model(lowercase			,  attention_mask=lowercase			,  past_key_values=lowercase  )[0]

								self.parent.assertEqual(next_tokens.shape[1]			,  output_from_past.shape[1]  )

								# select random slice
								UpperCAmelCase			:   List[str]      =				int(ids_tensor((1,)			,  output_from_past.shape[-1]  )  )
								UpperCAmelCase			:   Optional[int]      =				output_from_no_past[:, -3:, random_slice_idx]
								UpperCAmelCase			:   Optional[Any]      =				output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(lowercase			,  lowercase			,  rtol=1E-3  )



def   lowercase_	(    _lowercase       :		Optional[Any]							,      _lowercase       :		Optional[int]							,      _lowercase       :		Tuple							,      _lowercase       :		Tuple=None							,      _lowercase       :		Any=None							,      _lowercase       :		str=None							,      _lowercase       :		Optional[Any]=None							,      _lowercase       :		Dict=None							,      ):




					'''simple docstring'''






					if attention_mask is None:
										UpperCAmelCase			:   List[str]      =				tf.cast(tf.math.not_equal(_lowercase							,      config.pad_token_id	)							,      tf.inta	)
					if decoder_attention_mask is None:
										UpperCAmelCase			:   str      =				tf.concat(
										    [
										        tf.ones(decoder_input_ids[:, :1].shape							,      dtype=tf.inta	),
										        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]							,      config.pad_token_id	)							,      tf.inta	),
										    ]							,      axis=-1							,      )
					if head_mask is None:
										UpperCAmelCase			:   Dict      =				tf.ones((config.encoder_layers, config.encoder_attention_heads)	)
					if decoder_head_mask is None:
										UpperCAmelCase			:   Optional[Any]      =				tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
					if cross_attn_head_mask is None:
										UpperCAmelCase			:   str      =				tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
					return {
					    "input_ids": input_ids,
					    "decoder_input_ids": decoder_input_ids,
					    "attention_mask": attention_mask,
					    "decoder_attention_mask": decoder_attention_mask,
					    "head_mask": head_mask,
					    "decoder_head_mask": decoder_head_mask,
					    "cross_attn_head_mask": cross_attn_head_mask,
					}







@require_tf
class       snake_case__  ( lowerCAmelCase_					,  lowerCAmelCase_					,  unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						(
			    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
			)
			SCREAMING_SNAKE_CASE__     				=						(TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
			SCREAMING_SNAKE_CASE__     				=						(
			    {
			        '''conversational''': TFBlenderbotSmallForConditionalGeneration,
			        '''feature-extraction''': TFBlenderbotSmallModel,
			        '''summarization''': TFBlenderbotSmallForConditionalGeneration,
			        '''text2text-generation''': TFBlenderbotSmallForConditionalGeneration,
			        '''translation''': TFBlenderbotSmallForConditionalGeneration,
			    }
			    if is_tf_available()
			    else {}
			)
			SCREAMING_SNAKE_CASE__     				=						True
			SCREAMING_SNAKE_CASE__     				=						False
			SCREAMING_SNAKE_CASE__     				=						False







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				TFBlenderbotSmallModelTester(self  )
								UpperCAmelCase			:   Dict      =				ConfigTester(self			,  config_class=lowercase  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								self.config_tester.run_common_tests()







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*lowercase  )







@require_tokenizers
@require_tf
class       snake_case__  ( unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						[
			    '''Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like '''
			    '''  i\'m going to throw up.\nand why is that?'''
			]
			SCREAMING_SNAKE_CASE__     				=						'''facebook/blenderbot_small-90M'''







			@cached_property
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								return BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M"  )







			@cached_property
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name  )
								return model







			@slow
			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				self.tokenizer(self.src_text			,  return_tensors="tf"  )
								UpperCAmelCase			:   List[Any]      =				self.model.generate(
								    model_inputs.input_ids			,  attention_mask=model_inputs.attention_mask			,  num_beams=2			,  use_cache=lowercase			,  )
								UpperCAmelCase			:   Dict      =				self.tokenizer.batch_decode(generated_ids.numpy()			,  skip_special_tokens=lowercase  )[0]
								assert generated_words in (
								    "i don't know. i just feel like i'm going to throw up. it's not fun.",
								    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
								    "i'm not sure. i just feel like i've been in a bad situation.",
								)


