

"""simple docstring"""






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast







class       snake_case__  ( unittest.TestCase     ):







			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				0







			@slow
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(lowercase  )
													self.assertIsNotNone(lowercase  )
													self.assertIsInstance(lowercase			,  (BertTokenizer, BertTokenizerFast)  )
													self.assertGreater(len(lowercase  )			,  0  )

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
													UpperCAmelCase			:   List[str]      =				AutoTokenizer.from_pretrained(lowercase  )
													self.assertIsNotNone(lowercase  )
													self.assertIsInstance(lowercase			,  (GPTaTokenizer, GPTaTokenizerFast)  )
													self.assertGreater(len(lowercase  )			,  0  )







			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(lowercase  )
								self.assertIsInstance(lowercase			,  (BertTokenizer, BertTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(lowercase  )
								self.assertIsInstance(lowercase			,  (RobertaTokenizer, RobertaTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  20  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				AutoConfig.from_pretrained(lowercase  )
								self.assertIsInstance(lowercase			,  lowercase  )
								# Check that tokenizer_type ≠ model_type
								UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(lowercase			,  config=lowercase  )
								self.assertIsInstance(lowercase			,  (BertTokenizer, BertTokenizerFast)  )
								self.assertEqual(tokenizer.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(lowercase			,  "vocab.txt"  )  )

													UpperCAmelCase			:   List[str]      =				AutoTokenizer.from_pretrained(lowercase			,  tokenizer_type="bert"			,  use_fast=lowercase  )
													self.assertIsInstance(lowercase			,  lowercase  )

								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(lowercase			,  "vocab.json"  )  )
													shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(lowercase			,  "merges.txt"  )  )

													UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained(lowercase			,  tokenizer_type="gpt2"			,  use_fast=lowercase  )
													self.assertIsInstance(lowercase			,  lowercase  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.txt"			,  os.path.join(lowercase			,  "vocab.txt"  )  )

													UpperCAmelCase			:   Any      =				AutoTokenizer.from_pretrained(lowercase			,  tokenizer_type="bert"  )
													self.assertIsInstance(lowercase			,  lowercase  )

								with tempfile.TemporaryDirectory() as tmp_dir:
													shutil.copy("./tests/fixtures/vocab.json"			,  os.path.join(lowercase			,  "vocab.json"  )  )
													shutil.copy("./tests/fixtures/merges.txt"			,  os.path.join(lowercase			,  "merges.txt"  )  )

													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained(lowercase			,  tokenizer_type="gpt2"  )
													self.assertIsInstance(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								with pytest.raises(lowercase  ):
													AutoTokenizer.from_pretrained("./"			,  tokenizer_type="xxx"  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
													UpperCAmelCase			:   int      =				tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"  )
													self.assertIsInstance(lowercase			,  (BertTokenizer, BertTokenizerFast)  )

													if isinstance(lowercase			,  lowercase  ):
																		self.assertEqual(tokenizer.basic_tokenizer.do_lower_case			,  lowercase  )
													else:
																		self.assertEqual(tokenizer.do_lower_case			,  lowercase  )

													self.assertEqual(tokenizer.model_max_length			,  5_12  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
													with self.assertRaisesRegex(
													    lowercase			,  "julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"			,  ):
																		UpperCAmelCase			:   List[str]      =				tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   str      =				TOKENIZER_MAPPING.values()
								UpperCAmelCase			:   Any      =				[]

								for slow_tok, fast_tok in tokenizers:
													if slow_tok is not None:
																		tokenizer_names.append(slow_tok.__name__  )

													if fast_tok is not None:
																		tokenizer_names.append(fast_tok.__name__  )

								for tokenizer_name in tokenizer_names:
													# must find the right class
													tokenizer_class_from_name(lowercase  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"			,  use_fast=lowercase  )			,  lowercase  )
								self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"  )			,  lowercase  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained("distilbert-base-uncased"			,  do_lower_case=lowercase  )
								UpperCAmelCase			:   List[Any]      =				"Hello, world. How are you?"
								UpperCAmelCase			:   Any      =				tokenizer.tokenize(lowercase  )
								self.assertEqual("[UNK]"			,  tokens[0]  )

								UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained("microsoft/mpnet-base"			,  do_lower_case=lowercase  )
								UpperCAmelCase			:   int      =				tokenizer.tokenize(lowercase  )
								self.assertEqual("[UNK]"			,  tokens[0]  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"  )
								self.assertEqual(type(lowercase  )			,  lowercase  )
								self.assertEqual(tokenizer.model_max_length			,  5_12  )
								self.assertEqual(tokenizer.vocab_size			,  3_00_00  )
								self.assertEqual(tokenizer.unk_token			,  "[UNK]"  )
								self.assertEqual(tokenizer.padding_side			,  "right"  )
								self.assertEqual(tokenizer.truncation_side			,  "right"  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained(lowercase  )
								self.assertIsInstance(lowercase			,  (BertTokenizer, BertTokenizerFast)  )
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(lowercase  )
													UpperCAmelCase			:   List[str]      =				AutoTokenizer.from_pretrained(lowercase  )

								self.assertIsInstance(lowercase			,  tokenizer.__class__  )
								self.assertEqual(tokenizera.vocab_size			,  12  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained("ctrl"  )
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				get_tokenizer_config("bert-base-cased"  )
								UpperCAmelCase			:   Tuple      =				config.pop("_commit_hash"			,  lowercase  )
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(lowercase			,  {"do_lower_case": False}  )

								# This model does not have a tokenizer_config so we get back an empty dict.
								UpperCAmelCase			:   Optional[Any]      =				get_tokenizer_config(lowercase  )
								self.assertDictEqual(lowercase			,  {}  )

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(lowercase  )
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(lowercase  )
													UpperCAmelCase			:   List[str]      =				get_tokenizer_config(lowercase  )

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["tokenizer_class"]			,  "BertTokenizer"  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								try:
													AutoConfig.register("custom"			,  lowercase  )

													AutoTokenizer.register(lowercase			,  slow_tokenizer_class=lowercase  )
													# Trying to register something existing in the Transformers library will raise an error
													with self.assertRaises(lowercase  ):
																		AutoTokenizer.register(lowercase			,  slow_tokenizer_class=lowercase  )

													UpperCAmelCase			:   Dict      =				CustomTokenizer.from_pretrained(lowercase  )
													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(lowercase  )

																		UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(lowercase  )
																		self.assertIsInstance(lowercase			,  lowercase  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								try:
													AutoConfig.register("custom"			,  lowercase  )

													# Can register in two steps
													AutoTokenizer.register(lowercase			,  slow_tokenizer_class=lowercase  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, None)  )
													AutoTokenizer.register(lowercase			,  fast_tokenizer_class=lowercase  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)  )

													del TOKENIZER_MAPPING._extra_content[CustomConfig]
													# Can register in one step
													AutoTokenizer.register(
													    lowercase			,  slow_tokenizer_class=lowercase			,  fast_tokenizer_class=lowercase  )
													self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,  (CustomTokenizer, CustomTokenizerFast)  )

													# Trying to register something existing in the Transformers library will raise an error
													with self.assertRaises(lowercase  ):
																		AutoTokenizer.register(lowercase			,  fast_tokenizer_class=lowercase  )

													# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
													# and that model does not have a tokenizer.json
													with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase			:   Optional[int]      =				BertTokenizerFast.from_pretrained(lowercase  )
																		bert_tokenizer.save_pretrained(lowercase  )
																		UpperCAmelCase			:   Union[str, Any]      =				CustomTokenizerFast.from_pretrained(lowercase  )

													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(lowercase  )

																		UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(lowercase  )
																		self.assertIsInstance(lowercase			,  lowercase  )

																		UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(lowercase			,  use_fast=lowercase  )
																		self.assertIsInstance(lowercase			,  lowercase  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								with self.assertRaises(lowercase  ):
													UpperCAmelCase			:   Any      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"  )
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(lowercase  ):
													UpperCAmelCase			:   Any      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase  )
								self.assertTrue(tokenizer.special_attribute_present  )
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(lowercase  )
													UpperCAmelCase			:   Dict      =				AutoTokenizer.from_pretrained(lowercase			,  trust_remote_code=lowercase  )
								self.assertTrue(reloaded_tokenizer.special_attribute_present  )

								if is_tokenizers_available():
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizerFast"  )

													# Test we can also load the slow version
													UpperCAmelCase			:   int      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase			,  use_fast=lowercase  )
													self.assertTrue(tokenizer.special_attribute_present  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													# Test tokenizer can be reloaded.
													with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(lowercase  )
																		UpperCAmelCase			:   str      =				AutoTokenizer.from_pretrained(lowercase			,  trust_remote_code=lowercase			,  use_fast=lowercase  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertTrue(reloaded_tokenizer.special_attribute_present  )
								else:
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertEqual(reloaded_tokenizer.__class__.__name__			,  "NewTokenizer"  )







			@require_tokenizers
			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								class       snake_case__  ( lowerCAmelCase_     ):
											SCREAMING_SNAKE_CASE__     				=						False







								class       snake_case__  ( lowerCAmelCase_     ):
											SCREAMING_SNAKE_CASE__     				=						NewTokenizer
											SCREAMING_SNAKE_CASE__     				=						False

								try:
													AutoConfig.register("custom"			,  lowercase  )
													AutoTokenizer.register(lowercase			,  slow_tokenizer_class=lowercase  )
													AutoTokenizer.register(lowercase			,  fast_tokenizer_class=lowercase  )
													# If remote code is not set, the default is to use local
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertFalse(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   Optional[int]      =				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"			,  use_fast=lowercase  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertFalse(tokenizer.special_attribute_present  )

													# If remote code is disabled, we load the local one.
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertFalse(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase			,  use_fast=lowercase  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertFalse(tokenizer.special_attribute_present  )

													# If remote is enabled, we load from the Hub
													UpperCAmelCase			:   List[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )
													self.assertTrue(tokenizer.special_attribute_present  )
													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer"			,  trust_remote_code=lowercase			,  use_fast=lowercase  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
													self.assertTrue(tokenizer.special_attribute_present  )

								finally:
													if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
													if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]







			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(
								    "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=lowercase  )
								self.assertTrue(tokenizer.special_attribute_present  )
								if is_tokenizers_available():
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizerFast"  )

													# Test we can also load the slow version
													UpperCAmelCase			:   Optional[Any]      =				AutoTokenizer.from_pretrained(
													    "hf-internal-testing/test_dynamic_tokenizer_legacy"			,  trust_remote_code=lowercase			,  use_fast=lowercase  )
													self.assertTrue(tokenizer.special_attribute_present  )
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )
								else:
													self.assertEqual(tokenizer.__class__.__name__			,  "NewTokenizer"  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								with self.assertRaisesRegex(
								    lowercase			,  "bert-base is not a local folder and is not a valid model identifier"  ):
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained("bert-base"  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								with self.assertRaisesRegex(
								    lowercase			,  r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"  ):
													UpperCAmelCase			:   Union[str, Any]      =				AutoTokenizer.from_pretrained(lowercase			,  revision="aaaaaa"  )







			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
								with RequestCounter() as counter:
													UpperCAmelCase			:   Tuple      =				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
													self.assertEqual(counter.get_request_count			,  0  )
													self.assertEqual(counter.head_request_count			,  1  )
													self.assertEqual(counter.other_request_count			,  0  )





"""simple docstring"""






def   lowercase_	(    _lowercase       :		str	):




					'''simple docstring'''






					if not all(x.isalpha() for x in string	):
										raise ValueError("String must only contain alphabetic characters."	)

					UpperCAmelCase			:   Optional[int]      =				sorted(string.lower()	)
					return len(_lowercase	) == len(set(_lowercase	)	)


if __name__ == "__main__":
	snake_case_   :   Tuple        =       input("""Enter a string """).strip()

	snake_case_   :   str        =       is_isogram(input_str)
	print(f'''{input_str} is {"an" if isogram else "not an"} isogram.''')


